Merge pull request #80 from storyblok/task/int-1075-starts-with-and-f…

…ilter-query feat(int-1075): Sync only certain pages from one space to the other
storyblok · Apr 2, 2024 · 71d5c4a · 71d5c4a
2 parents da0572a + d642ec8
commit 71d5c4a
Show file tree

Hide file tree

Showing 5 changed files with 126 additions and 62 deletions.
diff --git a/README.md b/README.md
@@ -250,6 +250,10 @@ $ storyblok sync --type <COMMAND> --source <SPACE_ID> --target <SPACE_ID>
 * `type`: describe the command type to execute. Can be: `folders`, `components`, `stories`, `datasources` or `roles`. It's possible pass multiple types separated by comma (`,`).
 * `source`: the source space to use to sync
 * `target`: the target space to use to sync
+* `starts-with`: sync only stories that starts with the given string
+* `filter`: sync stories based on the given filter. Required Options: Required options: `--keys`, `--operations`, `--values`
+* `keys`: Multiple keys should be separated by comma. Example: `--keys key1,key2`, `--keys key1`
+* `operations`: Operations to be used for filtering. Can be: `is`, `in`, `not_in`, `like`, `not_like`, `any_in_array`, `all_in_array`, `gt_date`, `lt_date`, `gt_int`, `lt_int`, `gt_float`, `lt_float`. Multiple operations should be separated by comma.
 
 #### Examples
 
@@ -260,6 +264,15 @@ $ storyblok sync --type components --source 00001 --target 00002
 # Sync components and stories from `00001` space to `00002` space
 $ storyblok sync --type components,stories --source 00001 --target 00002
 
+# Sync only stories that starts with `myStartsWithString` from `00001` space to `00002` space
+$ storyblok sync --type stories --source 00001 --target 00002 --starts-with myStartsWithString
+
+# Sync only stories with a category field like `reference` from `00001` space to `00002` space
+$ storyblok sync --type stories --source 00001 --target 00002 --filter --keys category --operations like --values reference
+
+# Sync only stories with a category field like `reference` and a name field not like `demo` from `00001` space to `00002` space
+$ storyblok sync --type stories --source 00001 --target 00002 --filter --keys category,name --operations like,not_like --values reference,demo
+
 ```
 
 ### quickstart

diff --git a/src/cli.ts b/src/cli.ts
@@ -9,7 +9,7 @@ import { ALL_REGIONS, EU_CODE, isRegion } from "@storyblok/region-helper";
 import updateNotifier from "update-notifier";
 import fs from "fs";
 import tasks from "./tasks";
-import { getQuestions, lastStep, api, creds } from "./utils";
+import { getQuestions, lastStep, api, creds, buildFilterQuery } from "./utils";
 import { SYNC_TYPES, COMMANDS } from "./constants";
 export * from "./types/index";
 import { dirname } from "node:path";
@@ -298,6 +298,11 @@ program
   )
   .requiredOption("--source <SPACE_ID>", "Source space id")
   .requiredOption("--target <SPACE_ID>", "Target space id")
+  .option('--starts-with <STARTS_WITH>', 'Sync only stories that starts with the given string')
+  .option('--filter', 'Enable filter options to sync only stories that match the given filter. Required options: --keys; --operations; --values')
+  .option('--keys <KEYS>', 'Field names in your story object which should be used for filtering. Multiple keys should separated by comma.')
+  .option('--operations <OPERATIONS>', 'Operations to be used for filtering. Can be: is, in, not_in, like, not_like, any_in_array, all_in_array, gt_date, lt_date, gt_int, lt_int, gt_float, lt_float. Multiple operations should be separated by comma.')
+  .option('--values <VALUES>', 'Values to be used for filtering. Any string or number. If you want to use multiple values, separate them with a comma. Multiple values should be separated by comma.')
   .option("--components-groups <UUIDs>", "Synchronize components based on their group UUIDs separated by commas")
   .action(async (options) => {
     console.log(`${chalk.blue("-")} Sync data between spaces\n`);
@@ -307,10 +312,20 @@ program
         await api.processLogin();
       }
 
-      const { type, target, source, componentsGroups } = options;
+      const {
+        type,
+        target,
+        source, 
+        startsWith,
+        filter,
+        keys,
+        operations,
+        values,
+        componentsGroups
+      } = options;
 
       const _componentsGroups = componentsGroups ? componentsGroups.split(",") : null;
-
+      const filterQuery = filter ? buildFilterQuery(keys, operations, values) : undefined
       const token = creds.get().token || null;
 
       const _types = type.split(",") || [];
@@ -325,6 +340,8 @@ program
         token,
         target,
         source,
+        startsWith,
+        filterQuery,
         _componentsGroups,
       });
 

diff --git a/src/tasks/sync.js b/src/tasks/sync.js
@@ -11,11 +11,13 @@ const SyncSpaces = {
   init (options) {
     const { api } = options
     console.log(chalk.green('✓') + ' Loading options')
+    this.client = api.getClient()
     this.sourceSpaceId = options.source
     this.targetSpaceId = options.target
     this.oauthToken = options.token
-    this.client = api.getClient()
     this.componentsGroups = options._componentsGroups
+    this.startsWith = options.startsWith
+    this.filterQuery = options.filterQuery
   },
 
   async getStoryWithTranslatedSlugs (sourceStory, targetStory) {
@@ -42,93 +44,113 @@ const SyncSpaces = {
     return storyForPayload
   },
 
-  async syncStories () {
-    console.log(chalk.green('✓') + ' Syncing stories...')
+  async getTargetFolders () {
     const targetFolders = await this.client.getAll(`spaces/${this.targetSpaceId}/stories`, {
       folder_only: 1,
       sort_by: 'slug:asc'
     })
 
     const folderMapping = {}
 
-    for (let i = 0; i < targetFolders.length; i++) {
-      var folder = targetFolders[i]
+    for (const folder of targetFolders) {
       folderMapping[folder.full_slug] = folder.id
     }
 
-    const all = await this.client.getAll(`spaces/${this.sourceSpaceId}/stories`, {
-      story_only: 1
+    return folderMapping
+  },
+
+  async updateStoriesAndFolders (data, targetContent = null, sourceContent = null, isFolder = false) {
+    let createdStory = null
+    const contentName = sourceContent.name
+    const contentTypeName = isFolder ? 'Folder' : 'Story'
+    const payload = {
+      story: data,
+      force_update: '1',
+      ...(!isFolder && sourceContent.published ? { publish: 1 } : {})
+    }
+
+    if (targetContent) {
+      console.log(`${chalk.yellow('-')} ${contentTypeName} ${contentName} already exists`)
+      createdStory = await this.client.put(`spaces/${this.targetSpaceId}/stories/${targetContent.id}`, payload)
+      console.log(`${chalk.green('✓')} ${contentTypeName} ${targetContent.full_slug} updated`)
+    } else {
+      createdStory = await this.client.post(`spaces/${this.targetSpaceId}/stories`, payload)
+      console.log(`${chalk.green('✓')} ${contentTypeName} ${sourceContent.full_slug} created`)
+    }
+
+    createdStory = createdStory.data.story
+
+    if (createdStory.uuid !== sourceContent.uuid) {
+      await this.client.put(`spaces/${this.targetSpaceId}/stories/${createdStory.id}/update_uuid`, { uuid: sourceContent.uuid })
+    }
+
+    return createdStory
+  },
+
+  async syncStories () {
+    console.log(chalk.green('✓') + ' Syncing stories...')
+
+    const folderMapping = { ...await this.getTargetFolders() }
+
+    const allStories = await this.client.getAll(`spaces/${this.sourceSpaceId}/stories`, {
+      story_only: 1,
+      ...(this.startsWith ? { starts_with: this.startsWith } : {}),
+      ...(this.filterQuery ? { filter_query: this.filterQuery } : {})
     })
 
-    for (let i = 0; i < all.length; i++) {
-      console.log(chalk.green('✓') + ' Starting update ' + all[i].full_slug)
+    for (const story of allStories) {
+      console.log(chalk.green('✓') + ' Starting update ' + story.full_slug)
 
-      const { data } = await this.client.get('spaces/' + this.sourceSpaceId + '/stories/' + all[i].id)
+      const { data } = await this.client.get(`spaces/${this.sourceSpaceId}/stories/${story.id}`)
       const sourceStory = data.story
       const slugs = sourceStory.full_slug.split('/')
       let folderId = 0
 
       if (slugs.length > 1) {
         slugs.pop()
-        var folderSlug = slugs.join('/')
+        const folderSlug = slugs.join('/')
 
         if (folderMapping[folderSlug]) {
           folderId = folderMapping[folderSlug]
         } else {
-          console.error(chalk.red('X') + 'The folder does not exist ' + folderSlug)
+          console.error(`${chalk.red('X')} The folder does not exist ${folderSlug}`)
           continue
         }
       }
 
       sourceStory.parent_id = folderId
 
       try {
-        const existingStory = await this.client.get('spaces/' + this.targetSpaceId + '/stories', { with_slug: all[i].full_slug })
-        const storyData = await this.getStoryWithTranslatedSlugs(sourceStory, existingStory.data.stories ? existingStory.data.stories[0] : null)
-        const payload = {
-          story: storyData,
-          force_update: '1',
-          ...(sourceStory.published ? { publish: 1 } : {})
-        }
-
-        let createdStory = null
-        if (existingStory.data.stories.length === 1) {
-          createdStory = await this.client.put('spaces/' + this.targetSpaceId + '/stories/' + existingStory.data.stories[0].id, payload)
-          console.log(chalk.green('✓') + ' Updated ' + existingStory.data.stories[0].full_slug)
-        } else {
-          createdStory = await this.client.post('spaces/' + this.targetSpaceId + '/stories', payload)
-          console.log(chalk.green('✓') + ' Created ' + sourceStory.full_slug)
-        }
-        if (createdStory.data.story.uuid !== sourceStory.uuid) {
-          await this.client.put('spaces/' + this.targetSpaceId + '/stories/' + createdStory.data.story.id + '/update_uuid', { uuid: sourceStory.uuid })
-        }
+        const { data } = await this.client.get('spaces/' + this.targetSpaceId + '/stories', { with_slug: story.full_slug })
+        const existingStory = data.stories[0]
+        const storyData = await this.getStoryWithTranslatedSlugs(sourceStory, existingStory ? existingStory[0] : null)
+        await this.updateStoriesAndFolders(storyData, existingStory, sourceStory)
       } catch (e) {
         console.error(
-          chalk.red('X') + ` Story ${all[i].name} Sync failed: ${e.message}`
+          chalk.red('X') + ` Story ${story.name} Sync failed: ${e.message}`
         )
         console.log(e)
       }
     }
 
-    return Promise.resolve(all)
+    return Promise.resolve(allStories)
   },
 
   async syncFolders () {
     console.log(chalk.green('✓') + ' Syncing folders...')
+
     const sourceFolders = await this.client.getAll(`spaces/${this.sourceSpaceId}/stories`, {
       folder_only: 1,
       sort_by: 'slug:asc'
     })
     const syncedFolders = {}
 
-    for (var i = 0; i < sourceFolders.length; i++) {
-      const folder = sourceFolders[i]
-
+    for (const folder of sourceFolders) {
       try {
-        const folderResult = await this.client.get('spaces/' + this.sourceSpaceId + '/stories/' + folder.id)
-        const sourceFolder = folderResult.data.story
-        const existingFolder = await this.client.get('spaces/' + this.targetSpaceId + '/stories', { with_slug: folder.full_slug })
-        const folderData = await this.getStoryWithTranslatedSlugs(sourceFolder, existingFolder.data.stories ? existingFolder.data.stories[0] : null)
+        const folderResult = await this.client.get(`spaces/${this.sourceSpaceId}/stories/${folder.id}`)
+        const { data } = await this.client.get(`spaces/${this.targetSpaceId}/stories`, { with_slug: folder.full_slug })
+        const existingFolder = data.stories[0] || null
+        const folderData = await this.getStoryWithTranslatedSlugs(folderResult.data.story, existingFolder)
         delete folderData.id
         delete folderData.created_at
 
@@ -152,25 +174,7 @@ const SyncSpaces = {
           }
         }
 
-        const payload = {
-          story: folderData,
-          force_update: '1'
-        }
-
-        let createdFolder = null
-        if (existingFolder.data.stories.length === 1) {
-          console.log(`Folder ${folder.name} already exists`)
-          createdFolder = await this.client.put('spaces/' + this.targetSpaceId + '/stories/' + existingFolder.data.stories[0].id, payload)
-          console.log(chalk.green('✓') + ` Folder ${folder.name} updated`)
-        } else {
-          createdFolder = await this.client.post('spaces/' + this.targetSpaceId + '/stories', payload)
-          console.log(chalk.green('✓') + ` Folder ${folder.name} created`)
-        }
-        if (createdFolder.data.story.uuid !== folder.uuid) {
-          await this.client.put('spaces/' + this.targetSpaceId + '/stories/' + createdFolder.data.story.id + '/update_uuid', { uuid: folder.uuid })
-        }
-
-        syncedFolders[folder.id] = createdFolder.data.story.id
+        await this.updateStoriesAndFolders(folderData, existingFolder, folder, true)
       } catch (e) {
         console.error(
           chalk.red('X') + ` Folder ${folder.name} Sync failed: ${e.message}`

diff --git a/src/utils/build-filter-query.js b/src/utils/build-filter-query.js
@@ -0,0 +1,29 @@
+const buildFilterQuery = (keys, operations, values) => {
+  const operators = ['is', 'in', 'not_in', 'like', 'not_like', 'any_in_array', 'all_in_array', 'gt_date', 'lt_date', 'gt_int', 'lt_int', 'gt_float', 'lt_float']
+
+  if (!keys || !operations || !values) {
+    throw new Error('Filter options are required: --keys; --operations; --values')
+  }
+  const _keys = keys.split(',')
+  const _operations = operations.split(',')
+  const _values = values.split(',')
+
+  if (_keys.length !== _operations.length || _keys.length !== _values.length) {
+    throw new Error('The number of keys, operations and values must be the same')
+  }
+
+  const invalidOperators = _operations.filter((o) => !operators.includes(o))
+
+  if (invalidOperators.length) {
+    throw new Error('Invalid operator(s) applied for filter: ' + invalidOperators.join(' '))
+  }
+
+  const filterQuery = {}
+  _keys.forEach((key, index) => {
+    filterQuery[key] = { [_operations[index]]: _values[index] }
+  })
+
+  return filterQuery
+}
+
+export default buildFilterQuery
diff --git a/src/utils/index.js b/src/utils/index.js
@@ -7,3 +7,4 @@ export { default as findByProperty } from './find-by-property'
 export { default as parseError } from './parse-error'
 export { default as region } from './region'
 export { default as saveFileFactory } from './save-file-factory'
+export { default as buildFilterQuery } from './build-filter-query'