feat: use meta.json to request smarter & fix wrong initialization tim…

…ings of features (#717)

* feat: a `hasMeta` function with cache support to check if the meta file exists

* feat: now features that rely on OD data only run when meta.json exists

* refactor: encapsulate the meta related code with the MetaStore class

* feat: use meta.updatedAt to decide lastDataAvailableMonth

* fix: features initialize unexpectedly in the new UI

now they run expectedly in the both old and new UI

package.json

@@ -29,7 +29,7 @@
     "dom-loaded": "^3.0.0",
     "echarts": "^5.3.0",
     "element-ready": "^6.2.1",
-    "github-url-detection": "^6.1.0",
+    "github-url-detection": "^8.1.0",
     "jquery": "^3.6.0",
     "lodash-es": "^4.17.21",
     "office-ui-fabric-react": "^7.183.0",

src/api/common.ts

@@ -22,3 +22,81 @@ export const getMetricByName = async (
     }
   }
 };
+
+/**
+ * Common interface for both repo meta and user meta
+ * e.g. https://oss.x-lab.info/open_digger/github/X-lab2017/open-digger/meta.json (repo meta file)
+ * e.g. https://oss.x-lab.info/open_digger/github/tyn1998/meta.json (user meta file)
+ * @param name repo name or user name
+ */
+export interface CommonMeta {
+  type: 'user' | 'repo';
+  updatedAt: number; // time stamp
+  labels: unknown[]; // TODO: define the type
+}
+
+export interface RepoMeta extends CommonMeta {}
+
+export interface UserMeta extends CommonMeta {
+  repos: unknown[];
+}
+
+class MetaStore {
+  private static instance: MetaStore;
+  private responseCache: Map<string, Promise<Response>>;
+  private constructor() {
+    this.responseCache = new Map<string, Promise<Response>>();
+  }
+
+  public static getInstance(): MetaStore {
+    if (!MetaStore.instance) {
+      MetaStore.instance = new MetaStore();
+    }
+    return MetaStore.instance;
+  }
+
+  /**
+   * Fetch the meta file and cache the response
+   * @param name repo name or user name
+   */
+  private fetchMeta(name: string) {
+    const url = `${OSS_XLAB_ENDPOINT}/open_digger/github/${name}/meta.json`;
+    const promise = fetch(url);
+    this.responseCache.set(name, promise);
+  }
+
+  /**
+   * Check if the meta file exists
+   * @param name repo name or user name
+   * @returns true if the meta file exists, false otherwise
+   */
+  public async has(name: string) {
+    if (!this.responseCache.has(name)) {
+      this.fetchMeta(name);
+    }
+    const response = await this.responseCache.get(name)!;
+    if (!response.ok) {
+      return false;
+    } else {
+      return true;
+    }
+  }
+
+  /**
+   * Get the parsed meta file if it exists
+   * @param name repo name or user name
+   * @returns the parsed meta file if it exists, undefined otherwise
+   */
+  public async get(name: string): Promise<CommonMeta | undefined> {
+    if (await this.has(name)) {
+      const meta: CommonMeta = await this.responseCache
+        .get(name)!
+        // clone the response to avoid the response being used up
+        // https://stackoverflow.com/a/54115314/10369621
+        .then((res) => res.clone().json());
+      return meta;
+    }
+  }
+}
+
+export const metaStore = MetaStore.getInstance();

src/feature-manager.ts

@@ -125,7 +125,7 @@ const getFeatureID = (url: string): FeatureId => {
 /** Register a new feature */
 const add = async (
   id: FeatureId,
-  ...loaders: FeatureLoader[]
+  ...loaders: FeatureLoader[] // support multiple loaders for one feature, but currently only one is used
 ): Promise<void> => {
   /* Feature filtering and running */
   const options = await globalReady;

src/helpers/generate-data-by-month.ts

@@ -1,4 +1,11 @@
-const generateDataByMonth = (originalData: any) => {
+/**
+ * Months with value of 0 are not listed in data file for size optimization
+ * purpose, this function inserts those missing zeros.
+ * @param originalData
+ * @param updatedAt meta file last updated time
+ * @returns
+ */
+const generateDataByMonth = (originalData: any, updatedAt?: number) => {
   if (originalData === null) {
     return [];
   }
@@ -19,15 +26,16 @@ const generateDataByMonth = (originalData: any) => {
     else if (dateA > dateB) return 1;
     else return 0;
   });
+
+  // get the last month that has data
+  const lastDataAvailableMonth = updatedAt ? new Date(updatedAt) : new Date();
+  lastDataAvailableMonth.setDate(0);
+
   const oldestMonth = orderedMonths[0];
-  const now = new Date();
-  if (now.getDate() === 1) {
-    // data for last month is not ready in the first day of the month (#595)
-    now.setDate(0); // a way to let month - 1
-  }
-  now.setDate(0); // see issue #632
   const newestMonth =
-    now.getFullYear() + '-' + (now.getMonth() + 1).toString().padStart(2, '0');
+    lastDataAvailableMonth.getFullYear() +
+    '-' +
+    (lastDataAvailableMonth.getMonth() + 1).toString().padStart(2, '0');
   // insert no-event months (assigned to 0) and generate final data
   const arrayData: [string, number][] = [];
   const start = new Date(oldestMonth);

src/helpers/get-developer-info.ts

@@ -1,5 +1,14 @@
+import { metaStore } from '../api/common';
+
 import $ from 'jquery';
+import * as pageDetect from 'github-url-detection';
 
 export function getDeveloperName() {
   return $('.p-nickname.vcard-username.d-block').text().trim().split(' ')[0];
 }
+
+export async function isDeveloperWithMeta() {
+  return (
+    pageDetect.isUserProfile() && (await metaStore.has(getDeveloperName()))
+  );
+}

src/helpers/get-repo-info.ts

@@ -1,5 +1,33 @@
+import { metaStore } from '../api/common';
+
+import $ from 'jquery';
 import * as pageDetect from 'github-url-detection';
+import elementReady from 'element-ready';
 
 export function getRepoName() {
   return pageDetect.utils.getRepositoryInfo(window.location)!.nameWithOwner;
 }
+
+export function hasRepoContainerHeader() {
+  const headerElement = $('#repository-container-header');
+  return headerElement && !headerElement.attr('hidden');
+}
+
+export async function isRepoRoot() {
+  return pageDetect.isRepoRoot();
+}
+
+/**
+ * check if the repository is public
+ */
+export async function isPublicRepo() {
+  const selector = 'meta[name="octolytics-dimension-repository_public"]';
+  await elementReady(selector);
+  // <meta name="octolytics-dimension-repository_public" content="true/false">
+  const isPublic = $(selector).attr('content') === 'true';
+  return pageDetect.isRepo() && isPublic;
+}
+
+export async function isPublicRepoWithMeta() {
+  return (await isPublicRepo()) && (await metaStore.has(getRepoName()));
+}

src/helpers/is-perceptor.ts

@@ -1,4 +1,5 @@
-const isPerceptor = (): boolean =>
-  window.location.search.includes('?redirect=perceptor');
+const isPerceptor = (): boolean => {
+  return window.location.search.includes('?redirect=perceptor');
+};
 
 export default isPerceptor;

src/pages/ContentScripts/features/developer-activity-openrank-trends/index.tsx

@@ -1,25 +1,33 @@
 import React from 'react';
 import { render, Container } from 'react-dom';
 import $ from 'jquery';
-import * as pageDetect from 'github-url-detection';
 
 import features from '../../../../feature-manager';
-import { getDeveloperName } from '../../../../helpers/get-developer-info';
+import {
+  getDeveloperName,
+  isDeveloperWithMeta,
+} from '../../../../helpers/get-developer-info';
 import { getActivity, getOpenrank } from '../../../../api/developer';
+import { UserMeta, metaStore } from '../../../../api/common';
 import View from './view';
 
 const featureId = features.getFeatureID(import.meta.url);
 let developerName: string;
 let activity: any;
 let openrank: any;
+let meta: UserMeta;
 
 const getData = async () => {
   activity = await getActivity(developerName);
   openrank = await getOpenrank(developerName);
+  meta = (await metaStore.get(developerName)) as UserMeta;
 };
 
 const renderTo = (container: Container) => {
-  render(<View activity={activity} openrank={openrank} />, container);
+  render(
+    <View activity={activity} openrank={openrank} meta={meta} />,
+    container
+  );
 };
 
 const init = async (): Promise<void> => {
@@ -45,10 +53,9 @@ const restore = async () => {
   }
   renderTo($(`#${featureId}`)[0]);
 };
-
 features.add(featureId, {
-  include: [pageDetect.isUserProfile],
-  awaitDomReady: true,
+  asLongAs: [isDeveloperWithMeta],
+  awaitDomReady: false,
   init,
   restore,
 });

src/pages/ContentScripts/features/developer-activity-openrank-trends/view.tsx

@@ -8,22 +8,24 @@ import optionsStorage, {
   defaults,
 } from '../../../../options-storage';
 import Bars from '../../../../components/Bars';
+import { UserMeta } from '../../../../api/common';
 
 const githubTheme = getGithubTheme();
 
-const generateBarsData = (activity: any, openrank: any) => {
+const generateBarsData = (activity: any, openrank: any, updatedAt: number) => {
   return {
-    data1: generateDataByMonth(activity),
-    data2: generateDataByMonth(openrank),
+    data1: generateDataByMonth(activity, updatedAt),
+    data2: generateDataByMonth(openrank, updatedAt),
   };
 };
 
 interface Props {
   activity: any;
   openrank: any;
+  meta: UserMeta;
 }
 
-const View = ({ activity, openrank }: Props): JSX.Element | null => {
+const View = ({ activity, openrank, meta }: Props): JSX.Element | null => {
   const [options, setOptions] = useState<HypercrxOptions>(defaults);
 
   useEffect(() => {
@@ -34,7 +36,7 @@ const View = ({ activity, openrank }: Props): JSX.Element | null => {
 
   if (!activity || !openrank) return null;
 
-  let barsData: any = generateBarsData(activity, openrank);
+  let barsData: any = generateBarsData(activity, openrank, meta.updatedAt);
 
   return (
     <div className="border-top color-border-secondary pt-3 mt-3">

src/pages/ContentScripts/features/developer-networks/index.tsx

@@ -5,7 +5,10 @@ import * as pageDetect from 'github-url-detection';
 import elementReady from 'element-ready';
 
 import features from '../../../../feature-manager';
-import { getDeveloperName } from '../../../../helpers/get-developer-info';
+import {
+  getDeveloperName,
+  isDeveloperWithMeta,
+} from '../../../../helpers/get-developer-info';
 import { getDeveloperNetwork, getRepoNetwork } from '../../../../api/developer';
 import View from './view';
 
@@ -58,7 +61,7 @@ const restore = async () => {
 };
 
 features.add(featureId, {
-  asLongAs: [pageDetect.isUserProfile],
+  asLongAs: [isDeveloperWithMeta],
   awaitDomReady: false,
   init,
   restore,

src/pages/ContentScripts/features/oss-gpt/index.tsx

@@ -4,8 +4,7 @@ import $ from 'jquery';
 
 import features from '../../../../feature-manager';
 import getGithubTheme from '../../../../helpers/get-github-theme';
-import isPublicRepo from '../../../../helpers/is-public-repo';
-import { getRepoName } from '../../../../helpers/get-repo-info';
+import { getRepoName, isPublicRepo } from '../../../../helpers/get-repo-info';
 import View from './view';
 
 interface DocsMetaItem {

src/pages/ContentScripts/features/perceptor-layout/index.tsx

@@ -5,6 +5,7 @@ import { render, Container } from 'react-dom';
 
 import features from '../../../../feature-manager';
 import isPerceptor from '../../../../helpers/is-perceptor';
+import { isPublicRepoWithMeta } from '../../../../helpers/get-repo-info';
 import View from './view';
 
 const featureId = features.getFeatureID(import.meta.url);
@@ -33,7 +34,7 @@ const init = async (): Promise<void> => {
 };
 
 features.add(featureId, {
-  asLongAs: [isPerceptor],
+  asLongAs: [isPerceptor, isPublicRepoWithMeta],
   awaitDomReady: false,
   init,
 });

src/pages/ContentScripts/features/perceptor-tab/index.tsx

@@ -4,7 +4,7 @@ import elementReady from 'element-ready';
 import iconSvgPath from './icon-svg-path';
 import features from '../../../../feature-manager';
 import isPerceptor from '../../../../helpers/is-perceptor';
-import isPublicRepo from '../../../../helpers/is-public-repo';
+import { isPublicRepoWithMeta } from '../../../../helpers/get-repo-info';
 import sleep from '../../../../helpers/sleep';
 
 const featureId = features.getFeatureID(import.meta.url);
@@ -95,7 +95,7 @@ const init = async (): Promise<void> => {
 };
 
 features.add(featureId, {
-  asLongAs: [isPublicRepo],
+  asLongAs: [isPublicRepoWithMeta],
   awaitDomReady: false,
   init,
 });