feat: medusa-telemetry (#328)

* feat: adds a telemetry package to collect anonymous usage data

* fix: update telemetry host

* fix: adds medusa telemetry --disable

* fix: add tracking of link,login,new

* fix: interactively collect db credentials

* fix: require seed file

* fix: removes tracking from reporter

packages/medusa-telemetry/.babelrc.js

@@ -0,0 +1,12 @@
+let ignore = [`**/dist`]
+
+// Jest needs to compile this code, but generally we don't want this copied
+// to output folders
+if (process.env.NODE_ENV !== `test`) {
+  ignore.push(`**/__tests__`)
+}
+
+module.exports = {
+  presets: [["babel-preset-medusa-package"], ["@babel/preset-typescript"]],
+  ignore,
+}

packages/medusa-telemetry/.eslintrc

@@ -0,0 +1,9 @@
+{
+  "plugins": ["prettier"],
+  "extends": ["prettier"],
+  "rules": {
+    "prettier/prettier": "error",
+    "semi": "error",
+    "no-unused-expressions": "true"
+  }
+}

packages/medusa-telemetry/.gitignore

@@ -0,0 +1,6 @@
+/dist
+node_modules
+.DS_store
+.env*
+.env
+

packages/medusa-telemetry/.npmignore

@@ -0,0 +1,9 @@
+src
+.prettierrc
+.env
+.babelrc.js
+.eslintrc
+.gitignore
+ormconfig.json
+tsconfig.json
+jest.config.md

packages/medusa-telemetry/.prettierrc

@@ -0,0 +1,7 @@
+{
+  "endOfLine": "lf",
+  "semi": false,
+  "singleQuote": false,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

packages/medusa-telemetry/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "medusa-telemetry",
+  "version": "0.0.1",
+  "description": "Telemetry for Medusa",
+  "main": "dist/index.js",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/medusajs/medusa",
+    "directory": "packages/medusa-telemetry"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "author": "Sebastian Rindom",
+  "license": "MIT",
+  "devDependencies": {
+    "@babel/cli": "^7.14.3",
+    "@babel/core": "^7.14.3",
+    "@babel/preset-typescript": "^7.13.0",
+    "babel-preset-medusa-package": "^1.1.13",
+    "cross-env": "^5.2.1",
+    "eslint": "^6.8.0",
+    "jest": "^25.5.2",
+    "nodemon": "^2.0.1",
+    "prettier": "^1.19.1"
+  },
+  "scripts": {
+    "start": "nodemon --watch plugins/ --watch src/ --exec babel-node src/app.js",
+    "watch": "babel -w src --out-dir dist/ --ignore **/__tests__ --extensions \".ts,.js\"",
+    "prepare": "cross-env NODE_ENV=production npm run build",
+    "build": "babel src -d dist --ignore **/__tests__ --extensions \".ts,.js\"",
+    "serve": "node dist/app.js",
+    "postinstall": "node dist/postinstall.js || true",
+    "test": "jest",
+    "test:unit": "jest"
+  },
+  "dependencies": {
+    "axios": "^0.21.1",
+    "axios-retry": "^3.1.9",
+    "boxen": "^5.0.1",
+    "configstore": "5.0.1",
+    "is-docker": "^2.2.1",
+    "remove-trailing-slash": "^0.1.1",
+    "uuid": "^8.3.2"
+  }
+}

packages/medusa-telemetry/src/index.js

@@ -0,0 +1,20 @@
+import Telemeter from "./telemeter"
+import createFlush from "./util/create-flush"
+
+const telemeter = new Telemeter()
+
+export const flush = createFlush(telemeter.isTrackingEnabled())
+
+if (flush) {
+  process.on(`exit`, flush)
+}
+
+export const track = (event, data = {}) => {
+  telemeter.track(event, data)
+}
+
+export const setTelemetryEnabled = (enabled = true) => {
+  telemeter.setTelemetryEnabled(enabled)
+}
+
+export { default as Telemeter } from "./telemeter"

packages/medusa-telemetry/src/postinstall.js

@@ -0,0 +1,13 @@
+try {
+  const showAnalyticsNotification = require(`./util/show-notification`)
+  const Store = require(`./store`)
+
+  const eventStorage = new Store()
+  const disabled = eventStorage.disabled_
+  const enabledInConfig = eventStorage.getConfig(`telemetry.enabled`)
+  if (enabledInConfig === undefined && !disabled) {
+    showAnalyticsNotification()
+  }
+} catch (e) {
+  // ignore
+}

packages/medusa-telemetry/src/store.js

@@ -0,0 +1,59 @@
+import path from "path"
+import Configstore from "configstore"
+
+import InMemConfig from "./util/in-memory-config"
+import OutboxStore from "./util/outbox-store"
+import isTruthy from "./util/is-truthy"
+
+class Store {
+  constructor() {
+    try {
+      this.config_ = new Configstore(`medusa`, {}, { globalConfigPath: true })
+    } catch (e) {
+      this.config_ = new InMemConfig()
+    }
+
+    const baseDir = path.dirname(this.config_.path)
+    this.outbox_ = new OutboxStore(baseDir)
+
+    this.disabled_ = isTruthy(process.env.MEDUSA_DISABLE_TELEMETRY)
+  }
+
+  getQueueSize() {
+    return this.outbox_.getSize()
+  }
+
+  getQueueCount() {
+    return this.outbox_.getCount()
+  }
+
+  addEvent(event) {
+    if (this.disabled_) {
+      return
+    }
+
+    const eventString = JSON.stringify(event)
+    return this.outbox_.appendToBuffer(eventString + `\n`)
+  }
+
+  async flushEvents(handler) {
+    return await this.outbox_.startFlushEvents(async eventData => {
+      const events = eventData
+        .split(`\n`)
+        .filter(e => e && e.length > 2)
+        .map(e => JSON.parse(e))
+
+      return await handler(events)
+    })
+  }
+
+  getConfig(path) {
+    return this.config_.get(path)
+  }
+
+  setConfig(path, val) {
+    return this.config_.set(path, val)
+  }
+}
+
+export default Store

packages/medusa-telemetry/src/telemeter.js

@@ -0,0 +1,152 @@
+import os from "os"
+import fs from "fs"
+import { join, sep } from "path"
+import isDocker from "is-docker"
+import { v4 as uuidv4 } from "uuid"
+
+import createFlush from "./util/create-flush"
+import showAnalyticsNotification from "./util/show-notification"
+import isTruthy from "./util/is-truthy"
+import getTermProgram from "./util/get-term-program"
+import Store from "./store"
+
+const MEDUSA_TELEMETRY_VERBOSE = process.env.MEDUSA_TELEMETRY_VERBOSE || false
+
+class Telemeter {
+  constructor(options = {}) {
+    this.store_ = new Store()
+
+    this.flushAt = Math.max(options.flushAt, 1) || 20
+    this.maxQueueSize = options.maxQueueSize || 1024 * 500
+    this.flushInterval = options.flushInterval || 10 * 1000
+    this.flushed = false
+
+    this.queueSize_ = this.store_.getQueueSize()
+    this.queueCount_ = this.store_.getQueueCount()
+  }
+
+  getMachineId() {
+    if (this.machineId) {
+      return this.machineId
+    }
+    let machineId = this.store_.getConfig(`telemetry.machine_id`)
+    if (typeof machineId !== `string`) {
+      machineId = uuidv4()
+      this.store_.setConfig(`telemetry.machine_id`, machineId)
+    }
+    this.machineId = machineId
+    return machineId
+  }
+
+  isTrackingEnabled() {
+    // Cache the result
+    if (this.trackingEnabled !== undefined) {
+      return this.trackingEnabled
+    }
+    let enabled = this.store_.getConfig(`telemetry.enabled`)
+    if (enabled === undefined || enabled === null) {
+      showAnalyticsNotification()
+      enabled = true
+      this.store_.setConfig(`telemetry.enabled`, enabled)
+    }
+    this.trackingEnabled = enabled
+    return enabled
+  }
+
+  getOsInfo() {
+    if (this.osInfo) {
+      return this.osInfo
+    }
+    const cpus = os.cpus()
+    const osInfo = {
+      node_version: process.version,
+      platform: os.platform(),
+      release: os.release(),
+      cpus: (cpus && cpus.length > 0 && cpus[0].model) || undefined,
+      arch: os.arch(),
+      docker: isDocker(),
+      term_program: getTermProgram(),
+    }
+    this.osInfo = osInfo
+    return osInfo
+  }
+
+  getMedusaVersion() {
+    try {
+      const packageJson = require.resolve(`@medusajs/medusa/package.json`)
+      const { version } = JSON.parse(fs.readFileSync(packageJson, `utf-8`))
+      return version
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("failed to get medusa version", e)
+      }
+    }
+    return `-0.0.0`
+  }
+
+  getCliVersion() {
+    try {
+      const jsonfile = join(
+        require
+          .resolve(`@medusajs/medusa-cli`) // Resolve where current gatsby-cli would be loaded from.
+          .split(sep)
+          .slice(0, -2) // drop lib/index.js
+          .join(sep),
+        `package.json`
+      )
+
+      const { version } = require(jsonfile)
+      return version
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("failed to get medusa version", e)
+      }
+    }
+
+    return `-0.0.0`
+  }
+
+  setTelemetryEnabled(enabled) {
+    this.trackingEnabled = enabled
+    this.store_.setConfig(`telemetry.enabled`, enabled)
+  }
+
+  track(event, data) {
+    return this.enqueue_(event, data)
+  }
+
+  enqueue_(type, data) {
+    const event = {
+      id: `te_${uuidv4()}`,
+      type,
+      properties: data,
+      timestamp: new Date(),
+      machine_id: this.getMachineId(),
+      os_info: this.getOsInfo(),
+      medusa_version: this.getMedusaVersion(),
+      cli_version: this.getCliVersion(),
+    }
+
+    this.store_.addEvent(event)
+
+    this.queueCount_ += 1
+    this.queueSize_ += JSON.stringify(event).length
+
+    const hasReachedFlushAt = this.queueCount_ >= this.flushAt
+    const hasReachedQueueSize = this.queueSize_ >= this.maxQueueSize
+
+    if (hasReachedQueueSize || hasReachedFlushAt) {
+      const flush = createFlush(this.isTrackingEnabled())
+      flush && flush()
+    }
+
+    if (this.flushInterval && !this.timer) {
+      const flush = createFlush(this.isTrackingEnabled())
+      if (flush) {
+        this.timer = setTimeout(flush, this.flushInterval)
+      }
+    }
+  }
+}
+
+export default Telemeter

packages/medusa-telemetry/src/util/create-flush.js

@@ -0,0 +1,26 @@
+import { join } from "path"
+import { fork } from "child_process"
+import isTruthy from "./is-truthy"
+
+const MEDUSA_TELEMETRY_VERBOSE = process.env.MEDUSA_TELEMETRY_VERBOSE || false
+
+function createFlush(enabled) {
+  if (!enabled) {
+    return
+  }
+
+  return async function flush() {
+    if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+      console.log("Flushing queue...")
+    }
+
+    const forked = fork(join(__dirname, `send.js`), {
+      detached: true,
+      stdio: MEDUSA_TELEMETRY_VERBOSE ? `inherit` : `ignore`,
+      execArgv: [],
+    })
+    forked.unref()
+  }
+}
+
+export default createFlush

packages/medusa-telemetry/src/util/get-term-program.js

@@ -0,0 +1,12 @@
+function getTermProgram() {
+  const { TERM_PROGRAM, WT_SESSION } = process.env
+  if (TERM_PROGRAM) {
+    return TERM_PROGRAM
+  } else if (WT_SESSION) {
+    // https://github.com/microsoft/terminal/issues/1040
+    return `WindowsTerminal`
+  }
+  return undefined
+}
+
+export default getTermProgram

packages/medusa-telemetry/src/util/in-memory-config.js

@@ -0,0 +1,47 @@
+import { v4 as uuidv4 } from "uuid"
+import os from "os"
+import { join } from "path"
+
+export class InMemoryConfigStore {
+  config = {}
+  path = join(os.tmpdir(), `medusa`)
+
+  constructor() {
+    this.config = this.createBaseConfig()
+  }
+
+  createBaseConfig() {
+    return {
+      "telemetry.enabled": true,
+      "telemetry.machine_id": `not-a-machine-id-${uuidv4()}`,
+    }
+  }
+
+  get(key) {
+    return this.config[key]
+  }
+
+  set(key, value) {
+    this.config[key] = value
+  }
+
+  all() {
+    return this.config
+  }
+
+  size() {
+    return Object.keys(this.config).length
+  }
+
+  has(key) {
+    return !!this.config[key]
+  }
+
+  del(key) {
+    delete this.config[key]
+  }
+
+  clear() {
+    this.config = this.createBaseConfig()
+  }
+}

packages/medusa-telemetry/src/util/is-truthy.js

@@ -0,0 +1,23 @@
+// Returns true for `true`, true, positive numbers
+// Returns false for `false`, false, 0, negative integers and anything else
+function isTruthy(value) {
+  // Return if Boolean
+  if (typeof value === `boolean`) return value
+
+  // Return false if null or undefined
+  if (value === undefined || value === null) return false
+
+  // If the String is true or false
+  if (value.toLowerCase() === `true`) return true
+  if (value.toLowerCase() === `false`) return false
+
+  // Now check if it's a number
+  const number = parseInt(value, 10)
+  if (isNaN(number)) return false
+  if (number > 0) return true
+
+  // Default to false
+  return false
+}
+
+export default isTruthy

packages/medusa-telemetry/src/util/outbox-store.js

@@ -0,0 +1,121 @@
+import path from "path"
+import {
+  appendFileSync,
+  statSync,
+  readFileSync,
+  renameSync,
+  readdirSync,
+  existsSync,
+  unlinkSync,
+} from "fs"
+
+import isTruthy from "./is-truthy"
+
+const MEDUSA_TELEMETRY_VERBOSE = process.env.MEDUSA_TELEMETRY_VERBOSE || false
+
+class Outbox {
+  constructor(baseDir) {
+    this.eventsJsonFileName = `events.json`
+    this.bufferFilePath = path.join(baseDir, this.eventsJsonFileName)
+    this.baseDir = baseDir
+  }
+
+  appendToBuffer(event) {
+    try {
+      appendFileSync(this.bufferFilePath, event, `utf8`)
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("Failed to append to buffer", e)
+      }
+    }
+  }
+
+  getSize() {
+    if (!existsSync(this.bufferFilePath)) {
+      return 0
+    }
+
+    try {
+      const stats = statSync(this.bufferFilePath)
+      return stats.size
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("Failed to get outbox size", e)
+      }
+    }
+    return 0
+  }
+
+  getCount() {
+    if (!existsSync(this.bufferFilePath)) {
+      return 0
+    }
+
+    try {
+      const fileBuffer = readFileSync(this.bufferFilePath)
+      const str = fileBuffer.toString()
+      const lines = str.split("\n")
+      return lines.length - 1
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("Failed to get outbox count", e)
+      }
+    }
+    return 0
+  }
+
+  async flushFile(filePath, flushOperation) {
+    const now = `${Date.now()}-${process.pid}`
+    let success = false
+    let contents = ``
+    try {
+      if (!existsSync(filePath)) {
+        return true
+      }
+      // Unique temporary file name across multiple concurrent Medusa instances
+      const newPath = `${this.bufferFilePath}-${now}`
+      renameSync(filePath, newPath)
+      contents = readFileSync(newPath, `utf8`)
+      unlinkSync(newPath)
+
+      // There is still a chance process dies while sending data and some events are lost
+      // This will be ok for now, however
+      success = await flushOperation(contents)
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("Failed to perform file flush", e)
+      }
+    } finally {
+      // if sending fails, we write the data back to the log
+      if (!success) {
+        if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+          console.error(
+            "File flush did not succeed - writing back to file",
+            success
+          )
+        }
+        this.appendToBuffer(contents)
+      }
+    }
+    return true
+  }
+
+  async startFlushEvents(flushOperation) {
+    try {
+      await this.flushFile(this.bufferFilePath, flushOperation)
+      const files = readdirSync(this.baseDir)
+      const filtered = files.filter(p => p.startsWith(`events.json`))
+      for (const file of filtered) {
+        await this.flushFile(path.join(this.baseDir, file), flushOperation)
+      }
+      return true
+    } catch (e) {
+      if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+        console.error("Failed to perform flush", e)
+      }
+    }
+    return false
+  }
+}
+
+export default Outbox

packages/medusa-telemetry/src/util/send.js

@@ -0,0 +1,10 @@
+import TelemetryDispatcher from "./telemetry-dispatcher"
+
+const MEDUSA_TELEMETRY_HOST = process.env.MEDUSA_TELEMETRY_HOST || ""
+const MEDUSA_TELEMETRY_PATH = process.env.MEDUSA_TELEMETRY_PATH || ""
+
+const dispatcher = new TelemetryDispatcher({
+  host: MEDUSA_TELEMETRY_HOST,
+  path: MEDUSA_TELEMETRY_PATH,
+})
+dispatcher.dispatch()

packages/medusa-telemetry/src/util/show-notification.js

@@ -0,0 +1,25 @@
+import boxen from "boxen"
+
+const defaultConfig = {
+  padding: 1,
+  borderColor: `blue`,
+  borderStyle: `double`,
+}
+
+const defaultMessage =
+  `Medusa collects anonymous usage analytics\n` +
+  `to help improve Medusa for all users.\n` +
+  `\n` +
+  `If you'd like to opt-out, you can use \`medusa telemetry --disable\`\n`
+
+/**
+ * Analytics notice for the end-user
+ */
+function showAnalyticsNotification(
+  config = defaultConfig,
+  message = defaultMessage
+) {
+  console.log(boxen(message, config))
+}
+
+export default showAnalyticsNotification

packages/medusa-telemetry/src/util/telemetry-dispatcher.js

@@ -0,0 +1,115 @@
+import removeSlash from "remove-trailing-slash"
+import axios from "axios"
+import axiosRetry from "axios-retry"
+
+import showAnalyticsNotification from "./show-notification"
+import Store from "../store"
+import isTruthy from "./is-truthy"
+
+const MEDUSA_TELEMETRY_VERBOSE = process.env.MEDUSA_TELEMETRY_VERBOSE || false
+
+class TelemetryDispatcher {
+  constructor(options) {
+    this.store_ = new Store()
+
+    this.host = removeSlash(
+      options.host || "https://telemetry.medusa-commerce.com"
+    )
+    this.path = removeSlash(options.path || "/batch")
+
+    let axiosInstance = options.axiosInstance
+    if (!axiosInstance) {
+      axiosInstance = axios.create()
+    }
+    this.axiosInstance = axiosInstance
+
+    this.timeout = options.timeout || false
+    this.flushed = false
+
+    axiosRetry(this.axiosInstance, {
+      retries: 3,
+      retryDelay: axiosRetry.exponentialDelay,
+      retryCondition: this.isErrorRetryable_,
+    })
+  }
+
+  isTrackingEnabled() {
+    // Cache the result
+    if (this.trackingEnabled !== undefined) {
+      return this.trackingEnabled
+    }
+    let enabled = this.store_.getConfig(`telemetry.enabled`)
+    if (enabled === undefined || enabled === null) {
+      showAnalyticsNotification()
+      enabled = true
+      this.store_.setConfig(`telemetry.enabled`, enabled)
+    }
+    this.trackingEnabled = enabled
+    return enabled
+  }
+
+  async dispatch() {
+    if (!this.isTrackingEnabled()) {
+      return
+    }
+
+    await this.store_.flushEvents(async events => {
+      if (!events.length) {
+        if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+          console.log("No events to POST - skipping")
+        }
+        return true
+      }
+
+      const data = {
+        batch: events,
+        timestamp: new Date(),
+      }
+
+      const req = {
+        headers: {},
+      }
+
+      return await this.axiosInstance
+        .post(`${this.host}${this.path}`, data, req)
+        .then(() => {
+          if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+            console.log("POSTing batch succeeded")
+          }
+          return true
+        })
+        .catch(e => {
+          if (isTruthy(MEDUSA_TELEMETRY_VERBOSE)) {
+            console.error("Failed to POST event batch", e)
+          }
+          return false
+        })
+    })
+  }
+
+  isErrorRetryable_(error) {
+    // Retry Network Errors.
+    if (axiosRetry.isNetworkError(error)) {
+      return true
+    }
+
+    if (!error.response) {
+      // Cannot determine if the request can be retried
+      return false
+    }
+
+    // Retry Server Errors (5xx).
+    if (error.response.status >= 500 && error.response.status <= 599) {
+      return true
+    }
+
+    // Retry if rate limited.
+    if (error.response.status === 429) {
+      return true
+    }
+
+    return false
+  }
+}
+
+export default TelemetryDispatcher