/* Data engineering visual lab */ /* Data Engineering Lab */ function DataEngineeringPage({ tweaks, onBackHome }) { const primary = tweaks.primaryColor; const [code, setCode] = React.useState(''); const [unlocked, setUnlocked] = React.useState(false); const [loading, setLoading] = React.useState(false); const [error, setError] = React.useState(''); const [conceptId, setConceptId] = React.useState('medallion'); const [activeStageId, setActiveStageId] = React.useState('bronze'); const [flowTick, setFlowTick] = React.useState(0); const [autoPlay, setAutoPlay] = React.useState(true); const concept = DE_CONCEPTS.find(item => item.id === conceptId) || DE_CONCEPTS[0]; const activeStage = concept.stages.find(stage => stage.id === activeStageId) || concept.stages[0]; const activeStageIndex = concept.stages.findIndex(stage => stage.id === activeStage.id); const unlock = async (candidateCode) => { const trimmedCode = candidateCode.trim(); setLoading(true); setError(''); try { await verifyJourneyCode(trimmedCode); sessionStorage.setItem(JOURNEY_CODE_STORAGE_KEY, trimmedCode); setCode(trimmedCode); setUnlocked(true); } catch (err) { sessionStorage.removeItem(JOURNEY_CODE_STORAGE_KEY); setUnlocked(false); setError(err.message === 'Invalid code' ? 'Code is not correct.' : err.message); } finally { setLoading(false); } }; React.useEffect(() => { const savedCode = sessionStorage.getItem(JOURNEY_CODE_STORAGE_KEY); if (savedCode) unlock(savedCode); }, []); React.useEffect(() => { const nextConcept = DE_CONCEPTS.find(item => item.id === conceptId) || DE_CONCEPTS[0]; setActiveStageId(nextConcept.stages[Math.min(1, nextConcept.stages.length - 1)].id); }, [conceptId]); React.useEffect(() => { if (!unlocked || !autoPlay) return undefined; const timer = window.setInterval(() => { setFlowTick(current => current + 1); }, 1000); return () => window.clearInterval(timer); }, [unlocked, autoPlay]); if (!unlocked) { return (
Private data engineering lab

DE Visuals

Uses the same 32-character Journey code to open interactive diagrams for medallion architecture and data engineering patterns.

{ event.preventDefault(); unlock(code); }} style={{ display: 'flex', flexDirection: 'column', gap: 14 }}> setCode(event.target.value)} placeholder="32-character secret code" autoComplete="off" spellCheck="false" style={{ width: '100%', padding: '14px 16px', borderRadius: 14, border: '1px solid rgba(var(--theme-text-rgb),0.1)', background: 'rgba(var(--theme-bg-rgb),0.44)', color: 'var(--theme-text)', fontFamily: 'Poppins, sans-serif', fontSize: 16, letterSpacing: '1px', outline: 'none', }} /> {error && (
{error}
)}
); } return (
Data engineering concepts

Dynamic DE Visuals

{concept.summary}

{DE_CONCEPTS.map(item => { const active = item.id === concept.id; return ( ); })}
{concept.eyebrow}

{concept.title}

{concept.question}
{concept.metrics.map(([label, value]) => (
{label}
{value}
))}
); } function DataFlowVisual({ concept, activeStageId, activeStageIndex, flowTick, color, onStageSelect }) { const isMedallion = concept.id === 'medallion'; if (concept.id === 'databricks_platform') { return ( ); } if (concept.id === 'managed_spark') { return ( ); } if (concept.id === 'sql_lakehouse') { return ( ); } if (isMedallion) { const flowStages = concept.stages.filter(stage => stage.id !== 'serving'); const servingStage = concept.stages.find(stage => stage.id === 'serving'); const servingActive = activeStageId === 'serving'; const servingPassed = activeStageIndex >= concept.stages.findIndex(stage => stage.id === 'serving'); return ( ); return (
{flowStages.map((stage, index) => { const stageIndex = concept.stages.findIndex(item => item.id === stage.id); const active = stage.id === activeStageId; const passed = stageIndex <= activeStageIndex; return ( onStageSelect(stage.id)} /> {index < flowStages.length - 1 && (
)}
); })}
{servingStage && ( <>
)}
); } return (
{Array.from({ length: 8 }).map((_, index) => { const left = (flowTick * 7 + index * 14) % 100; const top = 40 + ((index % 3) - 1) * 7; return ( ); })}
{concept.stages.map((stage, index) => { const active = stage.id === activeStageId; const passed = index <= activeStageIndex; return ( ); })}
); } function getDeStage(concept, id) { return concept.stages.find(stage => stage.id === id) || concept.stages[0]; } function WhiteboardStageCard({ stage, active, passed, onClick, children, style = {} }) { return ( ); } function WhiteboardArrow({ color, style = {}, flip = false }) { return ( ); } function DatabricksPlatformVisual({ concept, activeStageId, activeStageIndex, flowTick, color, onStageSelect }) { const workspace = getDeStage(concept, 'dbx_workspace'); const workloads = getDeStage(concept, 'dbx_workloads'); const catalog = getDeStage(concept, 'unity_catalog'); const lakehouse = getDeStage(concept, 'delta_lakehouse'); const compute = getDeStage(concept, 'managed_compute'); const stageIndex = (id) => concept.stages.findIndex(stage => stage.id === id); return (
onStageSelect(workloads.id)}>
{['SQL', 'AI/BI', 'Mosaic'].map(label => ( {label} ))}
onStageSelect(catalog.id)} style={{ display: 'grid', gridTemplateColumns: 'minmax(150px, 0.8fr) minmax(0, 1.4fr)', gap: 12, alignItems: 'center' }} >
{['Catalog', 'Schema', 'Lineage', 'Permissions'].map(item => ( {item} ))}
onStageSelect(lakehouse.id)} style={{ minHeight: 190 }} >
{['Bronze', 'Silver', 'Gold'].map((label, index) => (
{label}
))}
{Array.from({ length: 7 }).map((_, index) => { const left = 12 + ((flowTick * 5 + index * 13) % 76); const top = 24 + ((index * 17 + flowTick * 3) % 55); return ( ); })}
); } function ManagedSparkProjectsVisual({ concept, activeStageId, activeStageIndex, flowTick, color, onStageSelect }) { const stageIndex = (id) => concept.stages.findIndex(stage => stage.id === id); const builders = getDeStage(concept, 'builders'); const projects = getDeStage(concept, 'dbx_projects'); const runtime = getDeStage(concept, 'spark_runtime'); const managed = getDeStage(concept, 'managed_cloud'); const operations = getDeStage(concept, 'operations'); return (
onStageSelect(builders.id)}> onStageSelect(projects.id)} style={{ minHeight: 164 }}>
{[0, 1, 2].map(i => )}
{['notebook.py', 'pipeline job', 'feature build'].map(item => ( ))}
onStageSelect(builders.id)}>
databricks
We manage it
{[0, 1, 2, 3].map(index => ( onStageSelect(runtime.id)} style={{ padding: 10 }}>
worker {index + 1}
))}
onStageSelect(managed.id)}>
cloud security + autoscale
onStageSelect(operations.id)}>
schedules + alerts
); } function SqlLakehouseVisual({ concept, activeStageId, activeStageIndex, flowTick, color, onStageSelect }) { const stageIndex = (id) => concept.stages.findIndex(stage => stage.id === id); const roles = getDeStage(concept, 'roles_to_tools'); const warehouse = getDeStage(concept, 'sql_warehouse'); const delta = getDeStage(concept, 'delta_lake'); const pipeline = getDeStage(concept, 'pipeline_build'); const ops = getDeStage(concept, 'managed_spark_ops'); return (
onStageSelect(warehouse.id)} style={{ minHeight: 150 }}>
SQL
onStageSelect(delta.id)} style={{ minHeight: 250 }}>
{['Bronze', 'Silver', 'Gold'].map((label, index) => (
{label}
))}
{[0, 1, 2, 3].map(index => { const top = 38 + ((flowTick * 5 + index * 17) % 32); return ; })}
); } function MedallionWhiteboardPlatform({ concept, flowStages, servingStage, activeStageId, activeStageIndex, flowTick, color, onStageSelect }) { const dataSamples = [ { system: 'ERP', source: 'ERP order #A17', bronze: 'raw_erp_orders', silver: 'clean_orders', gold: 'daily_revenue', }, { system: 'CRM', source: 'CRM contact', bronze: 'raw_crm_contacts', silver: 'dim_customer', gold: 'customer_360', }, { system: 'Web', source: 'web event', bronze: 'raw_clickstream', silver: 'session_events', gold: 'conversion_funnel', }, { system: 'API', source: 'inventory API', bronze: 'raw_inventory', silver: 'stock_snapshot', gold: 'sku_availability', }, { system: 'Finance', source: 'GL csv', bronze: 'raw_gl_entries', silver: 'clean_gl', gold: 'margin_kpi', }, ]; const progress = (flowTick * 7) % 100; const stageSize = 100 / flowStages.length; const currentStageIndex = Math.min(flowStages.length - 1, Math.floor(progress / stageSize)); const currentStage = flowStages[currentStageIndex]; const packetPosition = 7 + progress * 0.86; const packetIndex = Math.floor(flowTick / 2) % dataSamples.length; const currentPacket = dataSamples[packetIndex]; const visibleSamples = [0, 1, 2, 3].map(offset => dataSamples[(packetIndex + offset) % dataSamples.length]); const packetLabel = currentStage.id === 'sources' ? currentPacket.source : currentPacket[currentStage.id]; const servingActive = activeStageId === 'serving'; const servingPassed = activeStageIndex >= concept.stages.findIndex(stage => stage.id === 'serving'); return (
Unity Catalog
{['Schema', 'Catalog', 'Permissions'].map((item, index) => (
{item}
))}
Data Lakehouse
Medallion flow
{visibleSamples.map((sample, index) => ( {sample.source} ))}
stage.tone).join(', ')})`, }}/> {flowStages.map((stage, index) => ( ))}
{packetLabel} {currentPacket.system} -> {currentStage.label}
{flowStages.map(stage => { const stageIndex = concept.stages.findIndex(item => item.id === stage.id); const active = stage.id === activeStageId; const passed = stageIndex <= activeStageIndex; return ( ); })}
{servingStage && ( )}
{['ERP', 'CRM', 'API', 'Files'].map((item, index) => (
{item}
))}
); } function WhiteboardPerson({ title, subtitle, tone, icon }) { return (
{title} {subtitle}
); } function WhiteboardCapability({ title, subtitle, tone, icon }) { const iconPath = { warehouse: 'M5 19V8l7-4 7 4v11M8 19v-7h8v7M9 9h.01M12 9h.01M15 9h.01', bot: 'M8 11a4 4 0 018 0v4a4 4 0 01-8 0v-4zM9 7V4M15 7V4M6 13H4M20 13h-2M10 14h.01M14 14h.01', brain: 'M9 5a3 3 0 00-3 3v1a3 3 0 000 6v1a3 3 0 003 3M15 5a3 3 0 013 3v1a3 3 0 010 6v1a3 3 0 01-3 3M9 5v14M15 5v14M9 11h6M9 15h6', }[icon]; return (
{title} {subtitle}
); } function WhiteboardRole({ title, action, tone }) { return (
{title}
{action}
); } function SketchAvatar({ tone, small = false }) { const size = small ? 34 : 42; return ( ); } function SketchStackLogo({ color }) { return ( ); } function MiniServer({ tone }) { return ( ); } function MedallionTransformLane({ flowStages, flowTick, color }) { const dataSamples = [ { system: 'ERP', source: 'ERP order #A17', bronze: 'raw_erp_orders', silver: 'clean_orders', gold: 'daily_revenue', }, { system: 'CRM', source: 'CRM contact', bronze: 'raw_crm_contacts', silver: 'dim_customer', gold: 'customer_360', }, { system: 'Web', source: 'web event', bronze: 'raw_clickstream', silver: 'session_events', gold: 'conversion_funnel', }, { system: 'API', source: 'inventory API', bronze: 'raw_inventory', silver: 'stock_snapshot', gold: 'sku_availability', }, { system: 'Finance', source: 'GL csv', bronze: 'raw_gl_entries', silver: 'clean_gl', gold: 'margin_kpi', }, ]; const progress = (flowTick * 8) % 100; const packetPosition = 6 + progress * 0.88; const stageSize = 100 / flowStages.length; const currentStageIndex = Math.min(flowStages.length - 1, Math.floor(progress / stageSize)); const currentStage = flowStages[currentStageIndex]; const packetIndex = Math.floor(flowTick / 2) % dataSamples.length; const currentPacket = dataSamples[packetIndex]; const visibleSamples = [0, 1, 2].map(offset => dataSamples[(packetIndex + offset) % dataSamples.length]); const packetLabel = currentPacket[currentStage.id] || currentPacket.source; return (
Incoming {visibleSamples.map((sample, index) => ( {sample.source} ))}
stage.tone).join(', ')})`, opacity: 0.85, }}/> {flowStages.map((stage, index) => ( ))} {[18, 10].map((offset, index) => { const dotProgress = (progress - offset + 100) % 100; const dotPosition = 6 + dotProgress * 0.88; const dotStage = flowStages[Math.min(flowStages.length - 1, Math.floor(dotProgress / stageSize))]; return ( ); })}
{packetLabel} {currentPacket.system} -> {currentStage.label}
); } function MedallionLayerCard({ stage, index, active, passed, onClick }) { return ( ); } function DeConceptIcon({ conceptId, size = 19 }) { const paths = { medallion: [ 'M12 3l8 4-8 4-8-4 8-4z', 'M4 12l8 4 8-4', 'M4 17l8 4 8-4', ], databricks_platform: [ 'M6 8l6-3 6 3-6 3-6-3z', 'M6 13l6 3 6-3', 'M6 18l6 3 6-3', 'M18.5 5.5l2 2-2 2', ], managed_spark: [ 'M13 2L5 14h6l-1 8 8-12h-6l1-8z', 'M4 20h6M15 20h5', ], sql_lakehouse: [ 'M5 19V8l7-4 7 4v11', 'M8 19v-7h8v7', 'M5 22c3-2 5 1 8-1s5-1 7 1', ], powerbi: [ 'M4 19V5', 'M4 19h16', 'M8 16v-5M12 16V8M16 16v-7', ], orchestration: [ 'M5 7h5', 'M14 7h5', 'M10 7a2 2 0 104 0', 'M12 9v6', 'M7 17h10', ], quality: [ 'M12 3l8 4v5c0 5-3.4 8-8 9-4.6-1-8-4-8-9V7l8-4z', 'M8.5 12l2.2 2.2L16 9', ], }; const iconPaths = paths[conceptId] || paths.medallion; return ( ); } function DeStageIcon({ stageId, size = 19 }) { const paths = { sources: [ 'M4 7h16M4 12h16M4 17h16', 'M7 4v16M17 4v16', ], bronze: [ 'M12 3l8 4-8 4-8-4 8-4z', 'M4 12l8 4 8-4', 'M4 17l8 4 8-4', ], silver: [ 'M4 6h16v12H4z', 'M8 10h8M8 14h5', ], gold: [ 'M12 3l2.7 5.5 6.1.9-4.4 4.3 1 6.1L12 17.7 6.6 20.5l1-6.1-4.4-4.3 6.1-.9L12 3z', ], serving: [ 'M4 19V5', 'M4 19h16', 'M8 16v-5M12 16V8M16 16v-8', ], extract: [ 'M12 3v12', 'M7 10l5 5 5-5', 'M5 21h14', ], validate: [ 'M12 3l8 4v5c0 5-3.4 8-8 9-4.6-1-8-4-8-9V7l8-4z', 'M8.5 12l2.2 2.2L16 9', ], transform: [ 'M4 7h10', 'M10 3l4 4-4 4', 'M20 17H10', 'M14 13l-4 4 4 4', ], publish: [ 'M12 19V5', 'M7 10l5-5 5 5', 'M5 21h14', ], profile: [ 'M4 19V5M4 19h16', 'M8 16v-4M12 16V8M16 16v-7', ], assert: [ 'M5 12l4 4L19 6', 'M4 20h16', ], quarantine: [ 'M12 9v4', 'M12 17h.01', 'M10.3 4.3L2.8 18a2 2 0 001.7 3h15a2 2 0 001.7-3L13.7 4.3a2 2 0 00-3.4 0z', ], certify: [ 'M12 3l2.8 5.7 6.2.9-4.5 4.4 1.1 6.2L12 17.3 6.4 20.2 7.5 14 3 9.6l6.2-.9L12 3z', 'M9.5 12.2l1.7 1.7 3.4-4', ], dbx_workspace: [ 'M4 6h16v12H4z', 'M8 10h4M8 14h8', 'M16 6v12', ], dbx_workloads: [ 'M5 5h6v6H5z', 'M13 5h6v6h-6z', 'M5 13h6v6H5z', 'M13 13h6v6h-6z', ], unity_catalog: [ 'M4 6h16v12H4z', 'M8 6V4h8v2', 'M8 11h8M8 15h5', 'M17 14l2 2 3-4', ], delta_lakehouse: [ 'M5 16c3-4 6 2 9-2s5-3 7 1', 'M6 9l6-4 6 4-6 4-6-4z', 'M6 14l6 4 6-4', ], managed_compute: [ 'M4 8h16v5H4z', 'M4 16h16v5H4z', 'M17 10h.01M17 18h.01', 'M8 3v3M16 3v3', ], builders: [ 'M8 11a4 4 0 108 0', 'M4 21a8 8 0 0116 0', 'M17 4l3 3-3 3', ], dbx_projects: [ 'M3 7h7l2 2h9v10H3z', 'M7 13h10M7 16h6', ], spark_runtime: [ 'M13 2L5 14h6l-1 8 8-12h-6l1-8z', ], managed_cloud: [ 'M7 18a4 4 0 01.5-8A5 5 0 0117 8a4 4 0 011 8H7z', 'M9 18h9', 'M12 13l2 2 4-5', ], operations: [ 'M12 6v6l4 2', 'M21 12a9 9 0 11-2.6-6.4', 'M21 4v6h-6', ], roles_to_tools: [ 'M8 11a4 4 0 108 0', 'M4 21a8 8 0 0116 0', 'M5 5h4M15 5h4', ], sql_warehouse: [ 'M5 19V8l7-4 7 4v11', 'M8 19v-7h8v7', 'M9 9h.01M12 9h.01M15 9h.01', ], delta_lake: [ 'M5 16c3-4 6 2 9-2s5-3 7 1', 'M6 9l6-4 6 4-6 4-6-4z', 'M6 14l6 4 6-4', ], pipeline_build: [ 'M4 7h5', 'M15 7h5', 'M9 7a3 3 0 006 0', 'M4 17h5', 'M15 17h5', 'M9 17a3 3 0 016 0', ], managed_spark_ops: [ 'M13 2L5 14h6l-1 8 8-12h-6l1-8z', 'M18 16l2 2 3-4', ], pbi_sources: [ 'M4 7h16M4 12h16M4 17h16', 'M8 4v16M16 4v16', ], power_query: [ 'M5 5h14v5H5z', 'M7 14h10', 'M9 18h6', 'M18 10l3 3-3 3', ], semantic_model: [ 'M5 5h6v6H5z', 'M13 13h6v6h-6z', 'M11 8h4M15 8v5', ], dax_measures: [ 'M6 5h12', 'M8 19l8-14', 'M6 13h12', 'M6 19h12', ], report_canvas: [ 'M4 5h16v14H4z', 'M8 15v-4M12 15V8M16 15v-6', ], powerbi_service: [ 'M7 18a4 4 0 01.5-8A5 5 0 0117 8a4 4 0 011 8H7z', 'M9 18h9', 'M12 14l2 2 4-5', ], }; const iconPaths = paths[stageId] || paths.transform; return ( {iconPaths.map(path => )} ); } function DeDetailList({ title, items, color }) { return (
{title}
{items.map(item => (
{item}
))}
); }