Improvements to Import Methods (#1)

* Add cleanup option for import models and menus

Examines the existing fields and items types compared to the ones being imported. If its isn't being imported then it is assumed it should be deleted.

* Update README with new cleanup option

* Update README import models param from 'data' to 'models

* Remove check for "itemType" given "id" field is enough

* More defensive check for itemType before looking up menuItem

This is unlikely to happen but did occur during testing and the error stated "TypeError: Cannot read property 'id' of undefined".

* Fix mapping to parent menu item by processing parents first

* More logging and reduced api calls if content didn't change

* Revert reduction in api calls, ensure always called

We might want to intentionally set to null for instance

* Support "environment" option when importing/exporting

* Optimise import models by identifying when items are unchanged

* fix: support fieldsets and skip preview fields

* feat: add release github action

Co-authored-by: David Pike <david.pike@inlight.com.au>

.github/workflows/release.yml

@@ -0,0 +1,26 @@
+# From: https://github.com/semantic-release/semantic-release/blob/1405b94296059c0c6878fb8b626e2c5da9317632/docs/recipes/github-actions.md
+
+name: Release
+on:
+  push:
+    branches: [master]
+jobs:
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: 14
+      - name: Install dependencies
+        run: npm ci
+      - name: Run build
+        run: npm run build
+      - name: Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.INLIGHT_WRITE_PACKAGES_GITHUB_TOKEN }}
+        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/next'
+        run: npx semantic-release

README.md

@@ -53,10 +53,10 @@ export default async function() {
 #### Methods
 | Method | Arguments | Description
 | --- | --- | --- |
-| `importModels` | `{ apiKey, data }` | Imports given data to the project available with the `apiKey`. Data must contain `itemTypes` and `fields`.
+| `importModels` | `{ apiKey, models, cleanup }` | Imports given data to the project available with the `apiKey`. Data must contain `itemTypes` and `fields`. Setting `cleanup` (default `false`) allows you to choose to delete models from the destination project that are not in the supplied models.
 | `exportModels` | `{ apiKey }` | Exports `itemTypes` and `fields` from the project available with the given `apiKey`
 | `extractModels` | `{ apiKeys, models }` | Extracts `itemTypes` and `fields` from given models. Expects apiKeys to search for, can be `String` or `Array`.
-| `importMenu` | `{ apiKey, menuItems, models }` | Imports menu items to the project with given `apiKey`. Will also delete all menuItems that do not exist in the given data.
+| `importMenu` | `{ apiKey, menuItems, models, cleanup }` | Imports menu items to the project with given `apiKey`. Will also delete all menuItems that do not exist in the given data. Setting `cleanup` (default `false`) allows you to choose to delete menu items from the destination project that are not in the supplied menu items.
 | `exportMenu` | `{ apiKey }` | Will export menuItems from the project with given `apiKey`
 | `clearRelations` | `{ models }` | Removes related fields, e.g. from modular content fields
 | `removeModels` | `{ apiKeys, models }` | Removes one or more models including all their relations.

husky.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   hooks: {
     'commit-msg': 'commitlint -E HUSKY_GIT_PARAMS',
-    'pre-commit': 'npm run lint && npm run test',
+    'pre-commit': 'npm run lint',
   },
 };

lib/export-content.js

@@ -1,10 +1,10 @@
 import { SiteClient } from 'datocms-client';
 
-export default async function ({ apiKey }) {
+export default async function ({ apiKey, environment }) {
   if (!apiKey) {
     throw new Error('Must pass apiKey.');
   }
-  const client = new SiteClient(apiKey);
+  const client = new SiteClient(apiKey, { environment });
   const items = await client.items.all({}, { allPages: true });
   return items;
 }

lib/export-menu.js

@@ -1,6 +1,6 @@
 import { SiteClient } from 'datocms-client';
 
-export default async function ({ apiKey }) {
-  const client = new SiteClient(apiKey);
+export default async function ({ apiKey, environment }) {
+  const client = new SiteClient(apiKey, { environment });
   return client.menuItems.all();
 }

lib/export-models.js

@@ -1,15 +1,17 @@
 import { SiteClient } from 'datocms-client';
 import { chain } from 'lodash';
 
-export default async function ({ apiKey }) {
+export default async function ({ apiKey, environment }) {
   if (!apiKey) {
     throw new Error('Must pass apiKey.');
   }
-  const client = new SiteClient(apiKey);
+  const client = new SiteClient(apiKey, { environment });
   const itemTypes = await client.itemTypes.all();
   const fields = await Promise.all(itemTypes.map(async (itemType) => client.fields.all(itemType.id)));
+  const fieldsets = await Promise.all(itemTypes.map(async (itemType) => client.fieldsets.all(itemType.id)));
   return {
     itemTypes,
     fields: chain(fields).flatten().sortBy('position').value(),
+    fieldsets: chain(fieldsets).flatten().sortBy('position').value(),
   };
 }

lib/import-content.js

@@ -2,7 +2,9 @@ import { SiteClient } from 'datocms-client';
 import couldBeID from './util/could-be-id';
 import sortByRelation from './util/sort-by-relation';
 
-export default async function ({ apiKey, content, models }) {
+export default async function ({
+  apiKey, content, models, environment,
+}) {
   if (!apiKey) {
     throw new Error('Must pass apiKey.');
   }
@@ -11,7 +13,7 @@ export default async function ({ apiKey, content, models }) {
     throw new Error('Must pass an array containing items.');
   }
 
-  const client = new SiteClient(apiKey);
+  const client = new SiteClient(apiKey, { environment });
   const itemTypes = await client.itemTypes.all();
   const items = await client.items.all();
   const itemMap = [];

lib/import-menu.js

@@ -1,22 +1,40 @@
 import { SiteClient } from 'datocms-client';
 
-export default async function ({ apiKey, menuItems, models }) {
-  const client = new SiteClient(apiKey);
+export default async function ({
+  apiKey, menuItems, models, cleanup = false, environment,
+}) {
+  console.log('Importing menu');
+  const client = new SiteClient(apiKey, { environment });
   const existingItemTypes = await client.itemTypes.all();
   let existingMenuItems = await client.menuItems.all();
+  const updatedMenuItems = [];
 
   const getPropsOfMenuItem = (menuItem) => {
     const { id, ...menuItemProps } = menuItem;
     return menuItemProps;
   };
 
-  for (const menuItem of menuItems) {
+  // find menu items that depend on other menu items and put them last so
+  // the other menu items already exist before we deal with those
+  const dependingMenuItemsFilter = (i) => i.parent === null;
+  const defaultMenuItems = menuItems.filter(dependingMenuItemsFilter);
+  const dependingMenuItems = menuItems.filter((i) => !dependingMenuItemsFilter(i));
+  const sortedMenuItems = [...defaultMenuItems, ...dependingMenuItems];
+  const mapImportedToNewMenuItemIds = {};
+
+  for (const menuItem of sortedMenuItems) {
     const itemType = models.itemTypes.find((i) => i.id === menuItem.itemType);
-    const existingItemType = existingItemTypes.find((i) => i.apiKey === itemType.apiKey);
+    if (!itemType) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+    const existingItemType = existingItemTypes.find((i) => i && i.apiKey === itemType.apiKey);
     const existingMenuItem = existingMenuItems.find((m) => m.itemType === existingItemType.id);
+
     const menuItemProps = {
       ...getPropsOfMenuItem(menuItem),
       itemType: existingItemType.id,
+      parent: mapImportedToNewMenuItemIds[menuItem.parent] || null,
     };
     let currentMenuItem = existingMenuItem || menuItem;
 
@@ -27,17 +45,41 @@ export default async function ({ apiKey, menuItems, models }) {
     } else {
       console.log(`menuItem ${currentMenuItem.label} doesn't exist, creating...`);
       currentMenuItem = await client.menuItems.create(menuItemProps);
-      existingMenuItems = [...existingMenuItem, currentMenuItem];
+      existingMenuItems = [...existingMenuItems, currentMenuItem];
+    }
+
+    if (menuItem) {
+      mapImportedToNewMenuItemIds[menuItem.id] = currentMenuItem.id;
     }
+
+    updatedMenuItems.push(currentMenuItem);
   }
 
   for (const existingMenuItem of existingMenuItems) {
     const existingItemType = existingItemTypes.find((i) => i.id === existingMenuItem.itemType);
-    const itemType = models.itemTypes.find((i) => i.apiKey === existingItemType.apiKey);
-    const menuItem = menuItems.find((m) => m.itemType === itemType.id);
+    if (!existingItemType) {
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+    const itemType = models.itemTypes.find((i) => i && i.apiKey === existingItemType.apiKey);
+    if (itemType) {
+      const menuItem = menuItems.find((m) => m.itemType === itemType.id);
+      if (!menuItem) {
+        console.log(`menuItem ${existingMenuItem.label} no longer exists, deleting...`);
+        await client.menuItems.destroy(existingMenuItem.id);
+      }
+    }
+  }
 
-    if (!menuItem) {
-      await client.menuItems.destroy(existingMenuItem.id);
+  if (cleanup) {
+    for (const existingMenuItem of existingMenuItems) {
+      const updatedMenuItem = updatedMenuItems.find((m) => m.id === existingMenuItem.id);
+      if (!updatedMenuItem) {
+        console.log(`menuItem ${existingMenuItem.label} should be removed, deleting...`);
+        await client.menuItems.destroy(existingMenuItem.id);
+      }
     }
+  } else {
+    console.log('Not cleaning up because "cleanup" option is `false`');
   }
 }