Implement flexible expansion of parts of a large array

src/components/treemode/CollapsedItems.scss

@@ -0,0 +1,57 @@
+@import '../../styles.scss';
+
+$color: $gray;
+$background-color: $background-gray;
+
+div.collapsed-items {
+  font-family: $font-family;
+  font-size: $font-size;
+  color: $color;
+
+  // https://sharkcoder.com/visual/borders
+  $size: 8px;
+  padding: $padding / 2;
+  border: $size solid transparent;
+  border-width: $size 0;
+  background-color: $background-color;
+  background-color: hsla(0, 0%, 0%, 0);
+  background-image:
+      linear-gradient($background-color, $background-color),
+      linear-gradient(to bottom right, transparent 50.5%, $background-color 50.5%),
+      linear-gradient(to bottom left, transparent 50.5%, $background-color 50.5%),
+      linear-gradient(to top right, transparent 50.5%, $background-color 50.5%),
+      linear-gradient(to top left, transparent 50.5%, $background-color 50.5%);
+  background-repeat: repeat, repeat-x, repeat-x, repeat-x, repeat-x;
+  background-position: 0 0, $size 0, $size 0, $size 100%,$size 100%;
+  background-size: auto auto, 2*$size 2*$size, 2*$size 2*$size, 2*$size 2*$size, 2*$size 2*$size;
+  background-clip: padding-box, border-box, border-box, border-box, border-box;
+  background-origin: padding-box, border-box, border-box, border-box, border-box;
+
+  display: flex;
+
+  div.text,
+  button.expand-items {
+    margin: 0 $padding / 2;
+  }
+
+  div.text {
+    display: inline;
+  }
+
+  button.expand-items {
+    font-family: $font-family;
+    font-size: $font-size;
+    color: $color;
+
+    background: none;
+    border: none;
+    padding: 0;
+    text-decoration: underline;
+    cursor: pointer;
+
+    &:hover,
+    &:focus {
+      color: $red;
+    }
+  }
+}

src/components/treemode/CollapsedItems.svelte

@@ -0,0 +1,41 @@
+<script>
+  import {
+    INDENTATION_WIDTH
+  } from '../../constants.js'
+  import { getExpandItemsSections } from '../../logic/expandItemsSections.js'
+
+  export let visibleSections
+  export let sectionIndex
+  export let total
+  export let path
+
+  /** @type {function (path: Path, section: Section)} */
+  export let onExpandSection
+
+  $: visibleSection = visibleSections[sectionIndex]
+
+  $: startIndex = visibleSection.end
+  $: endIndex = visibleSections[sectionIndex + 1] 
+    ? visibleSections[sectionIndex + 1].start 
+    : total
+
+  $: expandItemsSections = getExpandItemsSections(startIndex, endIndex)
+
+  // TODO: this is duplicated from the same function in JSONNode
+  function getIndentationStyle(level) {
+    return `margin-left: ${level * INDENTATION_WIDTH}px`
+  }
+</script>
+
+<div class="collapsed-items" style={getIndentationStyle(path.length + 2)}>
+  <div>
+    <div class="text">Items {startIndex}-{endIndex}</div
+    >{#each expandItemsSections as expandItemsSection
+    }<button class="expand-items" on:click={() => onExpandSection(path, expandItemsSection)}>
+        show {expandItemsSection.start}-{expandItemsSection.end}
+      </button>
+    {/each}
+  </div>
+</div>
+
+<style src="./CollapsedItems.scss"></style>

src/components/treemode/JSONNode.scss

@@ -208,22 +208,6 @@
   }
 }
 
-div.limit {
-  button {
-    background: none;
-    border: none;
-    padding: 0;
-    color: $black;
-    text-decoration: underline;
-    cursor: pointer;
-
-    &:hover,
-    &:focus {
-      color: $red;
-    }
-  }
-}
-
 div.empty {
   &:not(:focus) {
     outline: 1px dotted lightgray;

src/components/treemode/JSONNode.svelte

@@ -6,12 +6,11 @@
   import { singleton } from './singleton.js'
   import {
     DEBOUNCE_DELAY,
-    DEFAULT_LIMIT,
     STATE_EXPANDED,
-    STATE_LIMIT,
     STATE_PROPS,
     STATE_SEARCH_PROPERTY,
     STATE_SEARCH_VALUE,
+    STATE_VISIBLE_SECTIONS,
     INDENTATION_WIDTH,
     VALIDATION_ERROR
   } from '../../constants.js'
@@ -29,6 +28,7 @@
   import { isUrl, stringConvert, valueType } from '../../utils/typeUtils'
   import { compileJSONPointer } from '../../utils/jsonPointer'
   import { getNextKeys } from '../../logic/documentState.js'
+  import CollapsedItems from './CollapsedItems.svelte'
 
   export let key = undefined // only applicable for object properties
   export let value
@@ -39,12 +39,15 @@
   export let onPatch
   export let onUpdateKey
   export let onExpand
-  export let onLimit
   export let onSelect
+  
+  /** @type {function (path: Path, section: Section)} */
+  export let onExpandSection
+
   export let selection
   
   $: expanded = state && state[STATE_EXPANDED]
-  $: limit = state && state[STATE_LIMIT]
+  $: visibleSections = state && state[STATE_VISIBLE_SECTIONS]
   $: props = state && state[STATE_PROPS]
   $: validationError = validationErrors && validationErrors[VALIDATION_ERROR]
 
@@ -56,6 +59,7 @@
 
   $: type = valueType (value)
 
+  $: limit = visibleSections && visibleSections[0].end // FIXME: make dynamic
   $: limited = type === 'array' && value.length > limit
 
   $: items = type === 'array'
@@ -211,14 +215,6 @@
     }
   }
 
-  function handleShowAll () {
-    onLimit(path, Infinity)
-  }
-
-  function handleShowMore () {
-    onLimit(path, (Math.round(limit / DEFAULT_LIMIT) + 1) * DEFAULT_LIMIT)
-  }
-
   function handleMouseDown (event) {
     // unselect existing selection on mouse down if any
     if (selection) {
@@ -400,21 +396,32 @@
     </div>
     {#if expanded}
       <div class="items">
-        {#each items as item, index (index)}
+        {#each visibleSections as visibleSection, sectionIndex (sectionIndex)}
-          <svelte:self
+          {#each value.slice(visibleSection.start, Math.min(visibleSection.end, value.length)) as item, itemIndex (itemIndex)}
-            key={index}
+            <svelte:self
-            value={item}
+              key={visibleSection.start + itemIndex}
-            path={path.concat(index)}
+              value={item}
-            state={state && state[index]}
+              path={path.concat(visibleSection.start + itemIndex)}
-            searchResult={searchResult ? searchResult[index] : undefined}
+              state={state && state[visibleSection.start + itemIndex]}
-            validationErrors={validationErrors ? validationErrors[index] : undefined}
+              searchResult={searchResult ? searchResult[visibleSection.start + itemIndex] : undefined}
-            onPatch={onPatch}
+              validationErrors={validationErrors ? validationErrors[visibleSection.start + itemIndex] : undefined}
-            onUpdateKey={handleUpdateKey}
+              onPatch={onPatch}
-            onExpand={onExpand}
+              onUpdateKey={handleUpdateKey}
-            onLimit={onLimit}
+              onExpand={onExpand}
-            onSelect={onSelect}
+              onSelect={onSelect}
-            selection={selection}
+              onExpandSection={onExpandSection}
-          />
+              selection={selection}
+            />
+          {/each}
+          {#if visibleSection.end < value.length}
+            <CollapsedItems 
+              visibleSections={visibleSections}
+              sectionIndex={sectionIndex}
+              total={value.length}
+              path={path}
+              onExpandSection={onExpandSection}
+            />
+          {/if}
         {/each}
         <div
           data-type="append-node-selector"
@@ -424,11 +431,11 @@
         >
           <div class="selector"></div>
         </div>
-        {#if limited}
+        <!-- {#if limited}
           <div class="limit" style={getIndentationStyle(path.length + 2)}>
             (showing {limit} of {value.length} items <button on:click={handleShowMore}>show more</button> <button on:click={handleShowAll}>show all</button>)
           </div>
-        {/if}
+        {/if} -->
       </div>
       <div data-type="selectable-area" class="footer" style={indentationStyle} >
         <span class="delimiter">]</span>
@@ -490,8 +497,8 @@
             onPatch={onPatch}
             onUpdateKey={handleUpdateKey}
             onExpand={onExpand}
-            onLimit={onLimit}
             onSelect={onSelect}
+            onExpandSection={onExpandSection}
             selection={selection}
           />
         {/each}

src/components/treemode/TreeMode.svelte

@@ -36,7 +36,7 @@ findRootPath
   import { immutableJSONPatch } from '../../utils/immutableJSONPatch'
   import { last, initial, cloneDeep, uniqueId, throttle } from 'lodash-es'
   import jump from '../../assets/jump.js/src/jump.js'
-  import { expandPath, syncState, patchProps } from '../../logic/documentState.js'
+  import { expandPath, expandSection, syncState, patchProps } from '../../logic/documentState.js'
   import Menu from './Menu.svelte'
   import { isObjectOrArray } from '../../utils/typeUtils.js'
   import { mapValidationErrors } from '../../logic/validation.js'
@@ -425,15 +425,6 @@ findRootPath
     }
   }
 
-  /**
-   * Change limit
-   * @param {Path} path
-   * @param {boolean} limit
-   */
-  function handleLimit (path, limit) {
-    state = setIn(state, path.concat(STATE_LIMIT), limit, true)
-  }
-
   /**
    * @param {SelectionSchema} selectionSchema
    */
@@ -463,6 +454,12 @@ findRootPath
     }
   }
 
+  function handleExpandSection (path, section) {
+    console.log('handleExpandSection', path, section)
+
+    state = expandSection(state, path, section)
+  }
+
   function handleKeyDown (event) {
     const combo = keyComboFromEvent(event)
 
@@ -579,8 +576,8 @@ findRootPath
       onPatch={handlePatch}
       onUpdateKey={handleUpdateKey}
       onExpand={handleExpand}
-      onLimit={handleLimit}
       onSelect={handleSelect}
+      onExpandSection={handleExpandSection}
       selection={selection}
     />
   </div>

src/constants.js

@@ -1,6 +1,7 @@
 
 export const STATE_EXPANDED = Symbol('expanded')
 export const STATE_LIMIT = Symbol('limit')
+export const STATE_VISIBLE_SECTIONS = Symbol('visible sections')
 export const STATE_PROPS = Symbol('props')
 export const STATE_SEARCH_PROPERTY = Symbol('search:property')
 export const STATE_SEARCH_VALUE = Symbol('search:value')
@@ -10,7 +11,8 @@ export const SCROLL_DURATION = 300 // ms
 export const DEBOUNCE_DELAY = 300
 export const SEARCH_PROGRESS_THROTTLE = 300 // ms
 export const MAX_SEARCH_RESULTS = 1000
-export const DEFAULT_LIMIT = 100
+export const ARRAY_SECTION_SIZE = 100
+export const DEFAULT_VISIBLE_SECTIONS = [{ start: 0, end: ARRAY_SECTION_SIZE }]
 export const MAX_PREVIEW_CHARACTERS = 20e3 // characters
 
 export const INDENTATION_WIDTH = 18 // pixels IMPORTANT: keep in sync with sass constant $indentation-width

src/logic/documentState.js

@@ -1,13 +1,14 @@
-import { initial, isEqual, isNumber, last, uniqueId } from 'lodash-es'
+import { initial, isEqual, isNumber, last, merge, uniqueId } from 'lodash-es'
 import {
-  DEFAULT_LIMIT,
+  DEFAULT_VISIBLE_SECTIONS,
   STATE_EXPANDED,
-  STATE_LIMIT,
+  STATE_PROPS,
-  STATE_PROPS
+  STATE_VISIBLE_SECTIONS
 } from '../constants.js'
-import { deleteIn, getIn, insertAt, setIn } from '../utils/immutabilityHelpers.js'
+import { deleteIn, getIn, insertAt, setIn, updateIn } from '../utils/immutabilityHelpers.js'
 import { parseJSONPointer } from '../utils/jsonPointer.js'
 import { isObject, isObjectOrArray } from '../utils/typeUtils.js'
+import { mergeSections, inVisibleSection, previousRoundNumber, nextRoundNumber } from './expandItemsSections.js'
 
 /**
  * Sync a state object with the doc it belongs to: update props, limit, and expanded state
@@ -52,19 +53,21 @@ export function syncState (doc, state = undefined, path, expand, forceRefresh =
       ? state[STATE_EXPANDED]
       : expand(path)
 
-    // note that we reset the limit when the state is not expanded
+    // note that we reset the visible items when the state is not expanded
-    updatedState[STATE_LIMIT] = (state && updatedState[STATE_EXPANDED])
+    updatedState[STATE_VISIBLE_SECTIONS] = (state && updatedState[STATE_EXPANDED])
-      ? state[STATE_LIMIT]
+      ? state[STATE_VISIBLE_SECTIONS]
-      : DEFAULT_LIMIT
+      : DEFAULT_VISIBLE_SECTIONS
 
     if (updatedState[STATE_EXPANDED]) {
-      for (let i = 0; i < Math.min(doc.length, updatedState[STATE_LIMIT]); i++) {
+      updatedState[STATE_VISIBLE_SECTIONS].forEach(({ start, end }) => {
-        const childDocument = doc[i]
+        for (let i = start; i < Math.min(doc.length, end); i++) {
-        if (isObjectOrArray(childDocument)) {
+          const childDocument = doc[i]
-          const childState = state && state[i]
+          if (isObjectOrArray(childDocument)) {
-          updatedState[i] = syncState(childDocument, childState, path.concat(i), expand, forceRefresh)
+            const childState = state && state[i]
+            updatedState[i] = syncState(childDocument, childState, path.concat(i), expand, forceRefresh)
+          }
         }
-      }
+      })
     }
 
     return updatedState
@@ -89,13 +92,17 @@ export function expandPath (state, path) {
     // FIXME: setIn has to create object first
     updatedState = setIn(updatedState, partialPath.concat(STATE_EXPANDED), true, true)
 
-    // if needed, enlarge the limit such that the search result becomes visible
+    // if needed, enlarge the expanded sections such that the search result becomes visible in the array
     const key = path[i]
     if (isNumber(key)) {
-      const limit = getIn(updatedState, partialPath.concat(STATE_LIMIT)) || DEFAULT_LIMIT
+      const sectionsPath = partialPath.concat(STATE_VISIBLE_SECTIONS)
-      if (key > limit) {
+      const sections = getIn(updatedState, sectionsPath) || DEFAULT_VISIBLE_SECTIONS
-        const newLimit = Math.ceil(key / DEFAULT_LIMIT) * DEFAULT_LIMIT
+      if (!inVisibleSection(sections, key)) {
-        updatedState = setIn(updatedState, partialPath.concat(STATE_LIMIT), newLimit, true)
+        const start = previousRoundNumber(key)
+        const end = nextRoundNumber(start)
+        const newSection = { start, end }
+        const updatedSections = mergeSections(sections.concat(newSection))
+        updatedState = setIn(updatedState, sectionsPath, updatedSections)
       }
     }
   }
@@ -103,6 +110,20 @@ export function expandPath (state, path) {
   return updatedState
 }
 
+/**
+ * Expand a section of items in an array
+ * @param {JSON} state
+ * @param {Path} path
+ * @param {Section} section
+ * @return {JSON} returns the updated state
+ */
+// TODO: write unit test
+export function expandSection (state, path, section) {
+  return updateIn(state, path.concat(STATE_VISIBLE_SECTIONS), (sections = DEFAULT_VISIBLE_SECTIONS) => {
+    return mergeSections(sections.concat(section))
+  })
+}
+
 export function updateProps (value, prevProps) {
   if (!isObject(value)) {
     return undefined

src/logic/documentState.test.js

@@ -1,9 +1,9 @@
 import assert from 'assert'
 import {
-  DEFAULT_LIMIT,
+  DEFAULT_VISIBLE_SECTIONS,
   STATE_EXPANDED,
-  STATE_LIMIT,
+  STATE_PROPS,
-  STATE_PROPS
+  STATE_VISIBLE_SECTIONS
 } from '../constants.js'
 import { syncState, updateProps } from './documentState.js'
 
@@ -30,7 +30,7 @@ describe('documentState', () => {
     ]
     expectedState.array = []
     expectedState.array[STATE_EXPANDED] = true
-    expectedState.array[STATE_LIMIT] = DEFAULT_LIMIT
+    expectedState.array[STATE_VISIBLE_SECTIONS] = DEFAULT_VISIBLE_SECTIONS
     expectedState.array[2] = {}
     expectedState.array[2][STATE_EXPANDED] = false
     expectedState.array[2][STATE_PROPS] = [

src/logic/expandItemsSections.js

@@ -0,0 +1,94 @@
+import { sortBy } from 'lodash-es'
+import { ARRAY_SECTION_SIZE } from '../constants.js'
+
+/**
+ * Create sections that can be expanded.
+ * Used to display a button like "Show items 100-200"
+ *
+ * @param {number} startIndex
+ * @param {number} endIndex
+ * @return {Section[]}
+ */
+export function getExpandItemsSections (startIndex, endIndex) {
+  // expand the start of the section
+  const section1 = {
+    start: startIndex,
+    end: Math.min(nextRoundNumber(startIndex), endIndex)
+  }
+
+  // expand the middle of the section
+  const start2 = Math.max(previousRoundNumber((startIndex + endIndex) / 2), startIndex)
+  const section2 = {
+    start: start2,
+    end: Math.min(nextRoundNumber(start2), endIndex)
+  }
+
+  // expand the end of the section
+  const section3 = {
+    start: Math.max(previousRoundNumber(endIndex), startIndex),
+    end: endIndex
+  }
+
+  const sections = [
+    section1
+  ]
+
+  const showSection2 = section2.start >= section1.end && section2.end <= section3.end
+  if (showSection2) {
+    sections.push(section2)
+  }
+
+  const showSection3 = section3.start >= (showSection2 ? section2.end : section1.end)
+  if (showSection3) {
+    sections.push(section3)
+  }
+
+  return sections
+}
+
+/**
+ * Sort and merge a list with sections
+ * @param {Section[]} sections
+ * @return {Section[]}
+ */
+export function mergeSections (sections) {
+  const sortedSections = sortBy(sections, section => section.start)
+
+  const mergedSections = [
+    sortedSections[0]
+  ]
+
+  for (let sortedIndex = 0; sortedIndex < sortedSections.length; sortedIndex++) {
+    const mergedIndex = mergedSections.length - 1
+    const previous = mergedSections[mergedIndex]
+    const current = sortedSections[sortedIndex]
+
+    if (current.start <= previous.end) {
+      // there is overlap -> replace the previous item
+      mergedSections[mergedIndex] = {
+        start: Math.min(previous.start, current.start),
+        end: Math.max(previous.end, current.end)
+      }
+    } else {
+      // no overlap, just add the item
+      mergedSections.push(current)
+    }
+  }
+
+  return mergedSections
+}
+
+// TODO: write unit test
+export function inVisibleSection (sections, index) {
+  return sections.some(section => {
+    return index >= section.start && index < section.end
+  })
+}
+
+export function nextRoundNumber (index) {
+  return Math.floor((index + ARRAY_SECTION_SIZE) / ARRAY_SECTION_SIZE) * ARRAY_SECTION_SIZE
+}
+
+export function previousRoundNumber (index) {
+  return Math.ceil((index - ARRAY_SECTION_SIZE) / ARRAY_SECTION_SIZE) * ARRAY_SECTION_SIZE
+}

src/logic/expandItemsSections.test.js

@@ -0,0 +1,113 @@
+import assert from 'assert'
+import {
+  mergeSections,
+  getExpandItemsSections,
+  nextRoundNumber,
+  previousRoundNumber
+} from './expandItemsSections.js'
+
+describe('expandItemsSections', () => {
+  it('should find the next round number', () => {
+    assert.strictEqual(nextRoundNumber(5), 100)
+    assert.strictEqual(nextRoundNumber(99), 100)
+    assert.strictEqual(nextRoundNumber(100), 200)
+  })
+
+  it('should find the previous round number', () => {
+    assert.strictEqual(previousRoundNumber(100), 0)
+    assert.strictEqual(previousRoundNumber(199), 100)
+    assert.strictEqual(previousRoundNumber(200), 100)
+    assert.strictEqual(previousRoundNumber(101), 100)
+    assert.strictEqual(previousRoundNumber(500), 400)
+  })
+
+  it('should calculate expandable sections (start, middle, end)', () => {
+    assert.deepStrictEqual(getExpandItemsSections(0, 1000), [
+      { start: 0, end: 100 },
+      { start: 400, end: 500 },
+      { start: 900, end: 1000 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 510), [
+      { start: 30, end: 100 },
+      { start: 200, end: 300 },
+      { start: 500, end: 510 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 250), [
+      { start: 30, end: 100 },
+      { start: 100, end: 200 },
+      { start: 200, end: 250 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 200), [
+      { start: 30, end: 100 },
+      { start: 100, end: 200 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 170), [
+      { start: 30, end: 100 },
+      { start: 100, end: 170 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 100), [
+      { start: 30, end: 100 }
+    ])
+
+    assert.deepStrictEqual(getExpandItemsSections(30, 70), [
+      { start: 30, end: 70 }
+    ])
+  })
+
+  it('should apply expanding a new piece of selection', () => {
+    // merge
+    assert.deepStrictEqual(mergeSections([
+      { start: 0, end: 100 },
+      { start: 100, end: 200 }
+    ]), [
+      { start: 0, end: 200 }
+    ])
+
+    // sort correctly
+    assert.deepStrictEqual(mergeSections([
+      { start: 0, end: 100 },
+      { start: 400, end: 500 },
+      { start: 200, end: 300 }
+    ]), [
+      { start: 0, end: 100 },
+      { start: 200, end: 300 },
+      { start: 400, end: 500 }
+    ])
+
+    // merge partial overlapping
+    assert.deepStrictEqual(mergeSections([
+      { start: 0, end: 30 },
+      { start: 20, end: 100 }
+    ]), [
+      { start: 0, end: 100 }
+    ])
+
+    // merge full overlapping
+    assert.deepStrictEqual(mergeSections([
+      { start: 100, end: 200 },
+      { start: 0, end: 300 }
+    ]), [
+      { start: 0, end: 300 }
+    ])
+    assert.deepStrictEqual(mergeSections([
+      { start: 0, end: 300 },
+      { start: 100, end: 200 }
+    ]), [
+      { start: 0, end: 300 }
+    ])
+
+    // merge overlapping with two
+    assert.deepStrictEqual(mergeSections([
+      { start: 0, end: 100 },
+      { start: 200, end: 300 },
+      { start: 100, end: 200 }
+    ]), [
+      { start: 0, end: 300 }
+    ])
+  })
+})

src/styles.scss

@@ -30,6 +30,9 @@ $hovered-background: #f0f0f0;
 $background-gray: #f5f5f5;
 $border-gray: #d8dbdf;
 
+$lightblue: lightblue;
+$darkblue: darkblue;
+
 $line-height: 18px;
 $indentation-width: 18px; // IMPORTANT: keep in sync with js constant INDENTATION_WIDTH
 $menu-button-size: 32px;

src/types.js

@@ -86,4 +86,8 @@
 
  /**
   * @typedef {{path: Path, message: string, isChildError?: boolean}} ValidationError
-  */
+  */
+  
+/**
+ * @typedef {{start: number, end: number}} Section
+ */