feat: bulk CIDs import (#2307)

MattWong-ca · lidel · semantic-release-bot · web-flow · commit 830a30d6c3ab · 2025-02-27T08:22:37.000-08:00
* Start bulk CID import * Get functional MVP working with csv file * Get it working with txt file * WIP: bulk import modal UX * WIP, need to debug modal UX * Modal UX working, need to clean up code * Remove comments * Clean code * Add validation to file select * Use localization * Add correct type, remove progress tracking * Refactor code * Add storybook file * Remove comments * Update Add menu test * Remove prop-types * Use functional component * Use useTranslation * Update to TS file * Update import path * fix(explore): browsing chunked files and inspecting via context menu (#2305) * fix(explore): chunked files This includes latest ipld-explorer-components with fix from ipfs/ipld-explorer-components#462 also bumped kubo and caniuse and non-breaking audit suggestions * fix(files): Inspect via context menu Closes #2306 * chore(ci): set cluster pin timeout to 30m https://github.com/ipfs/ipfs-webui/actions/workflows/ci.yml?page=4&query=is%3Asuccess are usually under 10-20 minutes if something takes longer it will likely take ages and then fail, so better to fail faster, allowing user to retry release * chore(ci): use repo in offline mode no need to start node and open outgoing connections github CI may be punishing us by throttling egress * Get functional MVP working with csv file * Get it working with txt file * WIP: bulk import modal UX * WIP, need to debug modal UX * Modal UX working, need to clean up code * Remove comments * Clean code * Add validation to file select * Use localization * Add correct type, remove progress tracking * Refactor code * Add storybook file * Remove comments * Update Add menu test * chore(release): 4.4.1 [skip ci] ## [4.4.1](v4.4.0...v4.4.1) (2024-11-30) CID `bafybeiatztgdllxnp5p6zu7bdwhjmozsmd7jprff4bdjqjljxtylitvss4` --- ### Bug Fixes * add lithuanian to languages.json ([#2293](#2293)) ([40c512b](40c512b)) * analyze script doesn't persist stats.json ([#2290](#2290)) ([dbbdd70](dbbdd70)) * **explore:** browsing chunked files and inspecting via context menu ([#2305](#2305)) ([0412970](0412970)), closes [#2306](#2306) ### Trivial Changes * **ci:** add CAR file directly to cluster ([#2304](#2304)) ([e2ae110](e2ae110)) * **ci:** no replication factor when pinning - use cluster's default ([#2302](#2302)) ([81b8f29](81b8f29)) * **ci:** set cluster pin timeout to 30m ([4b8fc00](4b8fc00)) * **ci:** udpate artifact actions to v4 ([#2292](#2292)) ([305908f](305908f)) * **ci:** use repo in offline mode ([eaf63ed](eaf63ed)) * pull new translations ([#2291](#2291)) ([bfe7e40](bfe7e40)) * pull transifex translations ([#2296](#2296)) ([502abd4](502abd4)) * pull transifex translations ([#2303](#2303)) ([89c094b](89c094b)) * size-related labels in Files screen ([#2295](#2295)) ([49019d4](49019d4)) * chore: pull new translations (#2308) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: lidel <lidel@users.noreply.github.com> * Remove prop-types * Use functional component * Use useTranslation * Update to TS file * Update import path --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: semantic-release-bot <semantic-release-bot@martynus.net> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: lidel <lidel@users.noreply.github.com> Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
diff --git a/public/locales/en/files.json b/public/locales/en/files.json
@@ -14,6 +14,7 @@
   "openWithLocalAndPublicGateway": "Try opening it instead with your <1>local gateway</1> or <3>public gateway</3>.",
   "cantBePreviewed": "Sorry, this file can’t be previewed",
   "addByPath": "From IPFS",
+  "bulkImport": "Bulk import",
   "newFolder": "New folder",
   "generating": "Generating…",
   "actions": {
@@ -59,6 +60,14 @@
     "namePlaceholder": "Name (optional)",
     "examples": "Examples:"
   },
+  "bulkImportModal": {
+    "title": "Bulk import with text file",
+    "description": "Upload a text file with a list of CIDs (names are optional). Example:",
+    "select": "Select file",
+    "selectedFile": "Selected file",
+    "invalidCids": "*Invalid CID(s) found",
+    "failedToReadFile": "*Failed to read file contents"
+  },
   "newFolderModal": {
     "title": "New folder",
     "description": "Insert the name of the folder you want to create."
diff --git a/src/bundles/files/actions.js b/src/bundles/files/actions.js
@@ -398,6 +398,53 @@ const actions = () => ({
     }
   }),
 
+  /**
+ * Reads a text file containing CIDs and adds each one to IPFS at the given root path.
+ * @param {FileStream[]} source - The text file containing CIDs
+ * @param {string} root - Destination directory in IPFS
+ */
+  doFilesBulkCidImport: (source, root) => perform(ACTIONS.BULK_CID_IMPORT, async function (ipfs, { store }) {
+    ensureMFS(store)
+
+    if (!source?.[0]?.content) {
+      console.error('Invalid file format provided to doFilesBulkCidImport')
+      return
+    }
+
+    try {
+      const file = source[0]
+      const content = await new Response(file.content).text()
+      const lines = content.split('\n').map(line => line.trim()).filter(Boolean)
+
+      const cidObjects = lines.map((line) => {
+        let actualCid = line
+        let name = line
+        const cidParts = line.split(' ')
+        if (cidParts.length > 1) {
+          actualCid = cidParts[0]
+          name = cidParts.slice(1).join(' ')
+        }
+        return {
+          name,
+          cid: actualCid
+        }
+      })
+
+      for (const { cid, name } of cidObjects) {
+        try {
+          const src = `/ipfs/${cid}`
+          const dst = realMfsPath(join(root || '/files', name || cid))
+
+          await ipfs.files.cp(src, dst)
+        } catch (err) {
+          console.error(`Failed to add CID ${cid}:`, err)
+        }
+      }
+    } finally {
+      await store.doFilesFetch()
+    }
+  }),
+
   /**
    * Creates a download link for the provided files.
    * @param {FileStat[]} files
diff --git a/src/bundles/files/consts.js b/src/bundles/files/consts.js
@@ -23,6 +23,8 @@ export const ACTIONS = {
   SHARE_LINK: ('FILES_SHARE_LINK'),
   /** @type {'FILES_ADDBYPATH'} */
   ADD_BY_PATH: ('FILES_ADDBYPATH'),
+  /** @type {'FILES_BULK_CID_IMPORT'} */
+  BULK_CID_IMPORT: ('FILES_BULK_CID_IMPORT'),
   /** @type {'FILES_PIN_ADD'} */
   PIN_ADD: ('FILES_PIN_ADD'),
   /** @type {'FILES_PIN_REMOVE'} */
diff --git a/src/bundles/files/protocol.ts b/src/bundles/files/protocol.ts
@@ -63,6 +63,7 @@ export type Message =
   | Move
   | Write
   | AddByPath
+  | BulkCidImport
   | DownloadLink
   | Perform<'FILES_SHARE_LINK', Error, string, void>
   | Perform<'FILES_COPY', Error, void, void>
@@ -76,6 +77,7 @@ export type MakeDir = Perform<'FILES_MAKEDIR', Error, void, void>
 export type WriteProgress = { paths: string[], progress: number }
 export type Write = Spawn<'FILES_WRITE', WriteProgress, Error, void, void>
 export type AddByPath = Perform<'FILES_ADDBYPATH', Error, void, void>
+export type BulkCidImport = Perform<'FILES_BULK_CID_IMPORT', Error, void, void>
 export type Move = Perform<'FILES_MOVE', Error, void, void>
 export type Delete = Perform<'FILES_DELETE', Error, void, void>
 export type DownloadLink = Perform<'FILES_DOWNLOADLINK', Error, FileDownload, void>
diff --git a/src/components/modal/Modal.js b/src/components/modal/Modal.js
@@ -2,7 +2,7 @@ import React from 'react'
 import PropTypes from 'prop-types'
 import CancelIcon from '../../icons/GlyphSmallCancel.js'
 
-export const ModalActions = ({ justify, className, children, ...props }) => (
+export const ModalActions = ({ justify = 'between', className = '', children, ...props }) => (
   <div className={`flex justify-${justify} pa2 ${className}`} style={{ backgroundColor: '#f4f6f8' }} {...props}>
     { children }
   </div>
@@ -13,12 +13,7 @@ ModalActions.propTypes = {
   className: PropTypes.string
 }
 
-ModalActions.defaultProps = {
-  justify: 'between',
-  className: ''
-}
-
-export const ModalBody = ({ className, Icon, title, children, ...props }) => (
+export const ModalBody = ({ className = '', Icon, title, children, ...props }) => (
   <div className={`ph4 pv3 tc ${className}`} {...props}>
     { Icon && (
       <div className='center bg-snow br-100 flex justify-center items-center' style={{ width: '80px', height: '80px' }}>
@@ -40,10 +35,6 @@ ModalBody.propTypes = {
   ])
 }
 
-ModalBody.defaultProps = {
-  className: ''
-}
-
 export const Modal = ({ onCancel, children, className, ...props }) => {
   return (
     <div className={`${className} bg-white w-80 shadow-4 sans-serif relative`} style={{ maxWidth: '34em' }} {...props}>
diff --git a/src/files/FilesPage.js b/src/files/FilesPage.js
@@ -15,13 +15,13 @@ import FilesList from './files-list/FilesList.js'
 import { getJoyrideLocales } from '../helpers/i8n.js'
 
 // Icons
-import Modals, { DELETE, NEW_FOLDER, SHARE, RENAME, ADD_BY_PATH, CLI_TUTOR_MODE, PINNING, PUBLISH } from './modals/Modals.js'
+import Modals, { DELETE, NEW_FOLDER, SHARE, RENAME, ADD_BY_PATH, BULK_CID_IMPORT, CLI_TUTOR_MODE, PINNING, PUBLISH } from './modals/Modals.js'
 import Header from './header/Header.js'
 import FileImportStatus from './file-import-status/FileImportStatus.js'
 import { useExplore } from 'ipld-explorer-components/providers'
 
 const FilesPage = ({
-  doFetchPinningServices, doFilesFetch, doPinsFetch, doFilesSizeGet, doFilesDownloadLink, doFilesDownloadCarLink, doFilesWrite, doFilesAddPath, doUpdateHash,
+  doFetchPinningServices, doFilesFetch, doPinsFetch, doFilesSizeGet, doFilesDownloadLink, doFilesDownloadCarLink, doFilesWrite, doFilesBulkCidImport, doFilesAddPath, doUpdateHash,
   doFilesUpdateSorting, doFilesNavigateTo, doFilesMove, doSetCliOptions, doFetchRemotePins, remotePins, pendingPins, failedPins,
   ipfsProvider, ipfsConnected, doFilesMakeDir, doFilesShareLink, doFilesDelete, doSetPinning, onRemotePinClick, doPublishIpnsKey,
   files, filesPathInfo, pinningServices, toursEnabled, handleJoyrideCallback, isCliTutorModeEnabled, cliOptions, t
@@ -72,6 +72,12 @@ const FilesPage = ({
     doFilesWrite(raw, root)
   }
 
+  const onBulkCidImport = (raw, root = '') => {
+    if (root === '') root = files.path
+
+    doFilesBulkCidImport(raw, root)
+  }
+
   const onAddByPath = (path, name) => doFilesAddPath(files.path, path, name)
   const onInspect = (cid) => doUpdateHash(`/explore/${cid}`)
   const showModal = (modal, files = null) => setModals({ show: modal, files })
@@ -206,6 +212,7 @@ const FilesPage = ({
         onAddFiles={onAddFiles}
         onMove={doFilesMove}
         onAddByPath={(files) => showModal(ADD_BY_PATH, files)}
+        onBulkCidImport={(files) => showModal(BULK_CID_IMPORT, files)}
         onNewFolder={(files) => showModal(NEW_FOLDER, files)}
         onCliTutorMode={() => showModal(CLI_TUTOR_MODE)}
         handleContextMenu={(...args) => handleContextMenu(...args, true)} />
@@ -226,6 +233,7 @@ const FilesPage = ({
         onShareLink={doFilesShareLink}
         onRemove={doFilesDelete}
         onAddByPath={onAddByPath}
+        onBulkCidImport={onBulkCidImport}
         onPinningSet={doSetPinning}
         onPublish={doPublishIpnsKey}
         cliOptions={cliOptions}
@@ -277,6 +285,7 @@ export default connect(
   'selectFilesSorting',
   'selectToursEnabled',
   'doFilesWrite',
+  'doFilesBulkCidImport',
   'doFilesDownloadLink',
   'doFilesDownloadCarLink',
   'doFilesSizeGet',
diff --git a/src/files/file-input/FileInput.js b/src/files/file-input/FileInput.js
@@ -50,6 +50,11 @@ class FileInput extends React.Component {
     this.toggleDropdown()
   }
 
+  onBulkCidImport = () => {
+    this.props.onBulkCidImport()
+    this.toggleDropdown()
+  }
+
   onNewFolder = () => {
     this.props.onNewFolder()
     this.toggleDropdown()
@@ -92,6 +97,10 @@ class FileInput extends React.Component {
               <NewFolderIcon className='fill-aqua w2 h2 mr1' />
               {t('newFolder')}
             </Option>
+            <Option onClick={this.onBulkCidImport} id='bulk-cid-import'>
+              <DocumentIcon className='fill-aqua w2 mr1' />
+              {t('bulkImport')}
+            </Option>
           </DropdownMenu>
         </Dropdown>
 
@@ -120,7 +129,8 @@ FileInput.propTypes = {
   t: PropTypes.func.isRequired,
   onAddFiles: PropTypes.func.isRequired,
   onAddByPath: PropTypes.func.isRequired,
-  onNewFolder: PropTypes.func.isRequired
+  onNewFolder: PropTypes.func.isRequired,
+  onBulkCidImport: PropTypes.func.isRequired
 }
 
 export default connect(
diff --git a/src/files/header/Header.js b/src/files/header/Header.js
@@ -93,6 +93,7 @@ class Header extends React.Component {
                   onNewFolder={this.props.onNewFolder}
                   onAddFiles={this.props.onAddFiles}
                   onAddByPath={this.props.onAddByPath}
+                  onBulkCidImport={this.props.onBulkCidImport}
                   onCliTutorMode={this.props.onCliTutorMode}
                 />
                 : <div ref={el => { this.dotsWrapper = el }}>
diff --git a/src/files/modals/Modals.js b/src/files/modals/Modals.js
@@ -10,6 +10,7 @@ import RenameModal from './rename-modal/RenameModal.js'
 import PinningModal from './pinning-modal/PinningModal.js'
 import RemoveModal from './remove-modal/RemoveModal.js'
 import AddByPathModal from './add-by-path-modal/AddByPathModal.js'
+import BulkImportModal from './bulk-import-modal/bulk-import-modal.tsx'
 import PublishModal from './publish-modal/PublishModal.js'
 import CliTutorMode from '../../components/cli-tutor-mode/CliTutorMode.js'
 import { cliCommandList, cliCmdKeys } from '../../bundles/files/consts.js'
@@ -20,6 +21,7 @@ const SHARE = 'share'
 const RENAME = 'rename'
 const DELETE = 'delete'
 const ADD_BY_PATH = 'add_by_path'
+const BULK_CID_IMPORT = 'bulk_cid_import'
 const CLI_TUTOR_MODE = 'cli_tutor_mode'
 const PINNING = 'pinning'
 const PUBLISH = 'publish'
@@ -30,6 +32,7 @@ export {
   RENAME,
   DELETE,
   ADD_BY_PATH,
+  BULK_CID_IMPORT,
   CLI_TUTOR_MODE,
   PINNING,
   PUBLISH
@@ -63,6 +66,11 @@ class Modals extends React.Component {
     this.leave()
   }
 
+  onBulkCidImport = (files, root) => {
+    this.props.onBulkCidImport(files, root)
+    this.leave()
+  }
+
   makeDir = (path) => {
     this.props.onMakeDir(join(this.props.root, path))
     this.leave()
@@ -152,6 +160,9 @@ class Modals extends React.Component {
       case ADD_BY_PATH:
         this.setState({ readyToShow: true })
         break
+      case BULK_CID_IMPORT:
+        this.setState({ readyToShow: true })
+        break
       case CLI_TUTOR_MODE:
         this.setState({ command: this.cliCommand(cliOptions, files, root) }, () => {
           this.setState({ readyToShow: true })
@@ -254,6 +265,13 @@ class Modals extends React.Component {
             onCancel={this.leave} />
         </Overlay>
 
+        <Overlay show={show === BULK_CID_IMPORT && readyToShow} onLeave={this.leave}>
+          <BulkImportModal
+            className='outline-0'
+            onBulkCidImport={this.onBulkCidImport}
+            onCancel={this.leave} />
+        </Overlay>
+
         <Overlay show={show === CLI_TUTOR_MODE && readyToShow} onLeave={this.leave}>
           <CliTutorMode onLeave={this.leave} filesPage={true} command={command} t={t}/>
         </Overlay>
diff --git a/src/files/modals/bulk-import-modal/bulk-import-modal.stories.js b/src/files/modals/bulk-import-modal/bulk-import-modal.stories.js
@@ -0,0 +1,21 @@
+import React from 'react'
+import { action } from '@storybook/addon-actions'
+import i18n from '../../../i18n-decorator.js'
+import BulkImportModal from './bulk-import-modal.tsx'
+
+/**
+ * @type {import('@storybook/react').Meta}
+ */
+export default {
+  title: 'Files/Modals',
+  decorators: [i18n]
+}
+
+/**
+ * @type {import('@storybook/react').StoryObj}
+ */
+export const BulkImport = () => (
+  <div className="ma3">
+    <BulkImportModal onCancel={action('Cancel')} onBulkCidImport={action('Bulk CID Import')} />
+  </div>
+)
diff --git a/src/files/modals/bulk-import-modal/bulk-import-modal.tsx b/src/files/modals/bulk-import-modal/bulk-import-modal.tsx
diff --git a/test/e2e/files.test.js b/test/e2e/files.test.js
