Usage reporting

frontend/editor/extended_python_generator.ts

@@ -130,6 +130,9 @@ export class OpModeDetails {
 // Extends the python generator to collect some information about functions and
 // variables that have been defined so they can be used in other modules.
 
+/** The resource name that the robot reports usage with, using wpiutil.report_usage. */
+const USAGE_REPORT_RESOURCE = 'Blocks';
+
 export class ExtendedPythonGenerator extends PythonGenerator {
   private readonly context: GeneratorContext;
   private workspace: Blockly.Workspace | null = null;
@@ -154,6 +157,10 @@ export class ExtendedPythonGenerator extends PythonGenerator {
   // All opmodes in the project (used when generating robot code)
   private allOpModeDetails: OpModeDetails[] = [];
 
+  // The usage report (as JSON) that the robot's __init__ sends with wpiutil.report_usage, or null
+  // if the robot shouldn't report usage.
+  private usageReportJson: string | null = null;
+
   private generateErrorHandling: boolean = false;
 
   constructor() {
@@ -220,6 +227,11 @@ export class ExtendedPythonGenerator extends PythonGenerator {
 
     switch (this.getModuleType()) {
       case storageModule.ModuleType.ROBOT:
+        if (this.usageReportJson !== null) {
+          this.importModule('wpiutil');
+          initStatements += this.INDENT + 'wpiutil.report_usage(' +
+              this.quote_(USAGE_REPORT_RESOURCE) + ', ' + this.quote_(this.usageReportJson) + ')\n';
+        }
         initStatements += this.INDENT + 'self.mechanisms = []\n';
         initStatements += this.INDENT + 'self.event_handlers = {}\n';
         initStatements += this.INDENT + 'self.define_hardware()\n';
@@ -652,6 +664,11 @@ export class ExtendedPythonGenerator extends PythonGenerator {
     this.allOpModeDetails = details;
   }
 
+  /** Sets the usage report (as JSON) for the robot to report, or null to not report usage. */
+  setUsageReportJson(usageReportJson: string | null) {
+    this.usageReportJson = usageReportJson;
+  }
+
   getClassName(): string {
     return this.context.getClassName();
   }

frontend/libraries/library_registry.ts

@@ -47,6 +47,11 @@ export function collectPythonData(libraries: Library[]): PythonData {
   return pythonData;
 }
 
+/** Returns all installed libraries. */
+export function getInstalledLibraries(): Library[] {
+  return [...librariesByName.values()];
+}
+
 /** Returns the installed library with the given name, or undefined if it isn't installed. */
 export function getInstalledLibrary(name: string): Library | undefined {
   return librariesByName.get(name);

frontend/storage/create_python_files.ts

@@ -12,9 +12,10 @@ import { extendedPythonGenerator } from '../editor/extended_python_generator';
 import { getOpModeDetailsFromBlocksJson } from '../blocks/mrc_opmode_details';
 import { Storage } from './common_storage';
 import { Module } from './module';
-import { parseModuleContentText } from './module_content';
+import { ModuleContent, parseModuleContentText } from './module_content';
 import { Project } from './project';
 import { pascalCaseToSnakeCase } from './names';
+import { createUsageReport } from './usage_report';
 import JSZip from 'jszip';
 
 /** Result of Python code generation for a single module */
@@ -35,17 +36,18 @@ interface ProjectPythonResult {
 
 /**
  * Generate Python code for a single module using headless Blockly
- * @param module The module containing Blockly JSON
- * @param storage The storage interface to fetch module content
+ * @param module The module
+ * @param moduleContent The content of the module, or an error if it couldn't be fetched
  * @returns Result containing generated Python code or error
  */
-async function generatePythonForModule(module: Module, storage: Storage): Promise<ModulePythonResult> {
+function generatePythonForModule(
+    module: Module, moduleContent: ModuleContent | Error): ModulePythonResult {
   const moduleName = pascalCaseToSnakeCase(module.className);
 
   try {
-    // Fetch the module content from storage
-    const moduleContentText = await storage.fetchFileContentText(module.modulePath);
-    const moduleContent = parseModuleContentText(moduleContentText);
+    if (moduleContent instanceof Error) {
+      throw moduleContent;
+    }
 
     // Create a headless workspace
     const workspace = workspaces.createHeadlessWorkspace(module.moduleType);
@@ -77,6 +79,26 @@ async function generatePythonForModule(module: Module, storage: Storage): Promis
   }
 }
 
+/**
+ * Fetches the content of each module in the project.
+ * @returns Maps the moduleId of each module to its content, or to the error if it couldn't be
+ * fetched or parsed.
+ */
+async function fetchModuleContents(
+    project: Project, storage: Storage): Promise<Map<string, ModuleContent | Error>> {
+  const modules: Module[] = [project.robot, ...project.mechanisms, ...project.opModes];
+  const contents = await Promise.all(modules.map(async (module): Promise<ModuleContent | Error> => {
+    try {
+      return parseModuleContentText(await storage.fetchFileContentText(module.modulePath));
+    } catch (error) {
+      return error instanceof Error ? error : new Error(String(error));
+    }
+  }));
+  const moduleContents = new Map<string, ModuleContent | Error>();
+  modules.forEach((module, i) => moduleContents.set(module.moduleId, contents[i]));
+  return moduleContents;
+}
+
 /**
  * Generate Python code for all modules in a project using headless Blockly
  * @param project The project containing multiple modules
@@ -87,42 +109,40 @@ async function generatePythonForProject(project: Project, storage: Storage): Pro
   const moduleResults: ModulePythonResult[] = [];
   let errorCount = 0;
 
+  const moduleContents = await fetchModuleContents(project, storage);
+  const validModuleContents = new Map<string, ModuleContent>();
+  for (const [moduleId, moduleContent] of moduleContents) {
+    if (!(moduleContent instanceof Error)) {
+      validModuleContents.set(moduleId, moduleContent);
+    }
+  }
+
   // Extract opmode details from all opmode modules and set on the generator
   // so the robot constructor can emit RegisterOpMode calls.
-  const allOpModeDetails = await Promise.all(
-    project.opModes.map(async (opMode) => {
-      const contentText = await storage.fetchFileContentText(opMode.modulePath);
-      const moduleContent = parseModuleContentText(contentText);
-      return getOpModeDetailsFromBlocksJson(moduleContent.getBlocks(), opMode.className);
-    })
-  );
+  const allOpModeDetails = project.opModes.map((opMode) => {
+    const moduleContent = moduleContents.get(opMode.moduleId);
+    if (!moduleContent || moduleContent instanceof Error) {
+      throw moduleContent || new Error(`Missing content for ${opMode.className}`);
+    }
+    return getOpModeDetailsFromBlocksJson(moduleContent.getBlocks(), opMode.className);
+  });
   extendedPythonGenerator.setAllOpModeDetails(
     allOpModeDetails.filter((d): d is NonNullable<typeof d> => d !== null)
   );
 
-  // Process the robot module
-  const robotResult = await generatePythonForModule(project.robot, storage);
-  moduleResults.push(robotResult);
-  if (!robotResult.success) {
-    errorCount++;
-  }
-
-  // Process all mechanism modules
-  for (const mechanism of project.mechanisms) {
-    const result = await generatePythonForModule(mechanism, storage);
-    moduleResults.push(result);
-    if (!result.success) {
-      errorCount++;
-    }
-  }
-
-  // Process all opmode modules
-  for (const opMode of project.opModes) {
-    const result = await generatePythonForModule(opMode, storage);
-    moduleResults.push(result);
-    if (!result.success) {
-      errorCount++;
+  // The robot code reports usage with wpiutil.report_usage.
+  extendedPythonGenerator.setUsageReportJson(
+      JSON.stringify(createUsageReport(project, validModuleContents)));
+  try {
+    for (const module of [project.robot, ...project.mechanisms, ...project.opModes]) {
+      const result = generatePythonForModule(module, moduleContents.get(module.moduleId)!);
+      moduleResults.push(result);
+      if (!result.success) {
+        errorCount++;
+      }
     }
+  } finally {
+    extendedPythonGenerator.setUsageReportJson(null);
   }
 
   return {

frontend/storage/usage_report.ts

@@ -0,0 +1,110 @@
+// Copyright (c) FIRST and other WPILib contributors.
+
+/**
+ * @fileoverview Builds the usage report that deployed robot code sends with
+ * wpiutil.report_usage('Blocks', ...).
+ */
+
+import { getBlocksVersion } from '../libraries/blocks_lib';
+import { getInstalledLibraries } from '../libraries/library_registry';
+import { ModuleContent, getTopLevelBlocksJson } from './module_content';
+import { Project } from './project';
+
+export interface UsageReport {
+  /** The version of blocks that generated the code. */
+  version: string;
+  /** Maps the name of each installed third party library to its version. */
+  libraries: {[name: string]: string};
+  /**
+   * Maps each block type used in the project to the number of times it is used. This also tells
+   * how many modules there are: the robot and each mechanism have one
+   * mrc_mechanism_component_holder block and each opmode has one mrc_opmode_details block.
+   */
+  blocks: {[blockType: string]: number};
+  /**
+   * Maps each component class (for example 'wpilib.ExpansionHubMotor') used by the robot to the
+   * number of those components. Components of a mechanism are counted once for each time the
+   * mechanism is added to the robot.
+   */
+  components: {[className: string]: number};
+}
+
+/**
+ * Builds the usage report for the given project.
+ * @param project The project.
+ * @param moduleContents Maps the moduleId of each module in the project to its content.
+ */
+export function createUsageReport(
+    project: Project, moduleContents: Map<string, ModuleContent>): UsageReport {
+  const libraries: {[name: string]: string} = {};
+  for (const library of getInstalledLibraries()) {
+    libraries[library.metadata.name] = library.metadata.version;
+  }
+
+  const blocks: {[blockType: string]: number} = {};
+  for (const moduleContent of moduleContents.values()) {
+    countBlocks(moduleContent.getBlocks(), blocks);
+  }
+
+  const components: {[className: string]: number} = {};
+  const robotContent = moduleContents.get(project.robot.moduleId);
+  if (robotContent) {
+    countComponents(robotContent, components);
+    for (const mechanismInRobot of robotContent.getMechanisms()) {
+      const mechanismContent = moduleContents.get(mechanismInRobot.moduleId);
+      if (mechanismContent) {
+        countComponents(mechanismContent, components);
+      }
+    }
+  }
+
+  return {
+    version: getBlocksVersion(),
+    libraries,
+    blocks: sortKeys(blocks),
+    components: sortKeys(components),
+  };
+}
+
+/**
+ * Adds the number of times each block type is used in the given blocks to counts. Shadow blocks
+ * are not counted, since they are put there by the toolbox rather than by the user.
+ */
+function countBlocks(blocks: {[key: string]: any}, counts: {[blockType: string]: number}): void {
+  const walk = (blockJson: {[key: string]: any}): void => {
+    if (!blockJson || typeof blockJson !== 'object') {
+      return;
+    }
+    if (typeof blockJson.type === 'string') {
+      counts[blockJson.type] = (counts[blockJson.type] || 0) + 1;
+    }
+    if (blockJson.inputs) {
+      for (const inputName in blockJson.inputs) {
+        const input = blockJson.inputs[inputName];
+        if (input && input.block) {
+          walk(input.block);
+        }
+      }
+    }
+    if (blockJson.next && blockJson.next.block) {
+      walk(blockJson.next.block);
+    }
+  };
+  getTopLevelBlocksJson(blocks).forEach(walk);
+}
+
+/** Adds the public and private components of the given module to counts. */
+function countComponents(
+    moduleContent: ModuleContent, counts: {[className: string]: number}): void {
+  for (const component of [...moduleContent.getComponents(), ...moduleContent.getPrivateComponents()]) {
+    counts[component.className] = (counts[component.className] || 0) + 1;
+  }
+}
+
+function sortKeys(counts: {[key: string]: number}): {[key: string]: number} {
+  const sorted: {[key: string]: number} = {};
+  for (const key of Object.keys(counts).sort()) {
+    sorted[key] = counts[key];
+  }
+  return sorted;
+}