2d:null 2e:["$","title",null,{"children":"짧은 머신러닝 안내서: UMAP | Humble Data Miner | Observable"}] 33:T90a,

## UMAP 알고리즘 개요 ${mermaid`%%{init: {'flowchart': {'htmlLabels': true}}}%% graph TB S0[("${steps[0].l.split(".")[1].split("(")[0]}")] S1["${steps[1].l.split("(")[0]} (local k-nearest-neighbor filtering)"] S2["${steps[2].l.split("(")[0]}(local membership strength computation)"] S3["${steps[3].l.split("(")[0]}(fuzzy union of membership strengths)"] S4["${steps[4].l.split("by")[0]}(Spectral Embedding)"] S5["${steps[5].l.split("by")[0]}(UMAP embedding optimization)"] T([최종 배치 결과]) S0A["원본 순차 데이터 (${dim.stk.length}×2779): ${dim.stk.length}종목의 가격 일변동 순차 데이터 원본 거리 데이터 (${dim.stk.length}×${dim.stk.length}): 원본 순차 데이터간의 거리 조합 # 이 예제에서는 distance metric으로 correlation distance를 사용 "] S2A["ρ_i←종목i 기준 최근점까지 거리 σ_i←membership-strength(종목i,종목i의 최근점)을 1로 만드는 분모값 membership-strength(종목i,종목j)=exp(-max{0,dist(종목i,종목j)-ρ_i}/σ_i) # 간단히 말해 각 종목마다 그 종목과 k근접 데이터들을 묶어 하나의 데이터 공간으로 가정하고 # 각 종목마다 상정된 데이터 공간 내에서 다른 점들을 연결강도를 0~1(최근점)사이로 재설정"] S3A["a←membership-strength(종목i,종목j) b←membership-strength(종목j,종목i) fuzzy-union(a,b)=a+b-a*b # probabilistic sum t-conorm"] S5A["forced directed graph 기반 재배치 방법"] S0--"${dim.stk.length}종목×${dim.stk.length}종목 (dense matrix)"-->S1 S1--"${dim.stk.length}종목×${dim.stk.length}종목 (sparse matrix)"-->S2 S2--"${dim.stk.length}종목×${dim.stk.length}종목 (sparse matrix)"-->S3 S3--"${dim.stk.length}종목×${dim.stk.length}종목 (sparse matrix)"-->S4 S4--"${dim.stk.length}종목×2D"-->S5 S5--"${dim.stk.length}종목×2D"-->T S0-.-S0A S2-.-S2A S3-.-S3A S5-.-S5A style S0A fill:lightyellow style S2A fill:lightyellow style S3A fill:lightyellow style S5A fill:lightyellow `}34:T53d,viewof params = { const sl_knn = d3_slider.sliderBottom() .min(0).max(presets.knn.length-1).step(1) .width(300) .value(presets.knn.indexOf(3)) .ticks(presets.knn.length) .tickFormat(i=>presets.knn[i]) .fill(d3.schemeCategory10[0]) .on("onchange", () => svg.dispatch("input")); const sl_min_dist = d3_slider.sliderBottom() .min(0).max(presets.min_dist.length-1).step(1) .width(300) .value(Math.floor(presets.min_dist.length/2)) .tickFormat(i=>presets.min_dist[i].toFixed(1)) .fill(d3.schemeCategory10[0]) .on("onchange", () => svg.dispatch("input")); const svg = d3.create("svg") .attr("viewBox", [0, 0, width, 60]); const g_sl_knn = svg.append("g").attr('transform', `translate(${40},${25})`); const g_sl_min_dist = svg.append("g").attr('transform', `translate(${40+360},${25})`); g_sl_knn.call(sl_knn); g_sl_knn.append("text").attr("transform",`translate(${300/2},${-8})`).attr("text-anchor","middle").text("# nearest neighbors"); g_sl_min_dist.call(sl_min_dist); g_sl_min_dist.append("text").attr("transform",`translate(${300/2},${-8})`).attr("text-anchor","middle").text("min dist (UMAP embedding 전용)"); return Object.defineProperty( svg.node(), "value", {get: () => ({knn_index:sl_knn.value(), min_dist_index:sl_min_dist.value()})} ); }35:T2f01,slide = { const gs = width/3; // grid-size const pm = {left:12,right:8,top:12+50,bottom:12}; const height = gs + pm.top + pm.bottom; const cs = (gs-pm.left-pm.right)/dim.stk.length; const svg = d3.create("svg").attr("viewBox", [0,0,width,height]); const ki = params.knn_index; const min_dist = presets.min_dist[params.min_dist_index]; const n_stks = dim.stk.length; const g_scroll = svg.append("g"); g_scroll.attr("transform",`translate(${-gs*input_data.pi},${0})`); const gx = d3.scaleLinear().domain([0,1]).range([pm.left,gs-pm.right]); function make_g_flat_graph(mat, interp_color, title, metric, ascend) { const coo_list = []; mat.forEach((v,i)=>{ coo_list.push({ri:i[0],ci:i[1],v:v}); },true); const coo_items = coo_list.reduce((acc,cur)=>{ if(!acc.hasOwnProperty(cur.ri)) acc[cur.ri] = []; acc[cur.ri].push(cur); return acc; },{}); const coo = d3.range(n_stks) .map(i=>coo_items[i].toSorted((a,b)=>ascend?d3.ascending(a.v,b.v):d3.descending(a.v,b.v))); coo.forEach((r,ri)=>{ r.forEach((d,i)=>{ d.off = -(r.length-1)/2+i; }) }); const g = g_scroll.append("g"); g.node().appendChild( Legend(d3.scaleSequential([0.0, 1.0], interp_color),{ title: metric, width:gs, marginLeft:pm.left, marginRight:pm.right })); const g_graph = g.append("g").attr("transform",`translate(${0},${cs/2})`); const g_grp = g_graph.selectAll("g") .data(coo) .join("g") .attr("transform",(d,ri)=>`translate(${0},${pm.top+cs*ri})`); const g_edge = g_grp.selectAll("path.edge") .data(d=>d) .join("path") .attr("stroke",d=>interp_color(d.v)) .attr("d",d=>`M ${gx(0)} ${d.off*2} H ${gx(d.v)}`); const g_lhs = g_grp.append("path").attr("class","lhs") .attr("transform",`translate(${gx(0)},${0})`) .attr("fill",(_,ri)=>dim.stk[ri].c) .attr("d",(_,ri)=>dim.stk[ri].sym()); g_lhs.append("title") .text((_,i)=>`(${dim.stk[i].sc}) ${dim.stk[i].sn}`); const g_rhs = g_grp.selectAll("path.rhs") .data(d=>d) .join("path") .attr("transform",d=>`translate(${gx(d.v)},${d.off*2})`) .attr("fill",d=>dim.stk[d.ci].c) .attr("d",d=>dim.stk[d.ci].sym()); g_rhs.append("title") .text(d=>`(${dim.stk[d.ci].sc}) ${dim.stk[d.ci].sn}`); const g_title = g.append("text") .attr("text-anchor","middle").attr("dominant-baseline","hanging") .attr("x",gs/2) .attr("y",height-pm.bottom-4) .text(title); return {g:g,g_grp:g_grp,g_lhs:g_lhs,g_rhs:g_rhs,g_edge:g_edge,g_title:g_title}; } function make_g_matrix(mat,interp_color,title,metric){ const data = mathjs.matrix(mat,'dense').map((v,i)=>({ri:i[0],ci:i[1],v:v})).toArray().flat(); const g = g_scroll.append("g"); g.node().appendChild( Legend(d3.scaleSequential([0.0, 1.0], interp_color),{ title: metric, width:gs, marginLeft:pm.left, marginRight:pm.right })); const g_rect = g.selectAll("path") .data(data) .join("path") .attr("d",d=>d.v>0? `M ${pm.left+d.ci*cs} ${pm.top+d.ri*cs} v ${cs} h ${cs} v ${-cs} Z`: `M ${pm.left+d.ci*cs} ${pm.top+d.ri*cs} v ${cs} h ${cs} v ${-cs} h ${-cs} l ${cs} ${cs} m ${0} ${-cs} l ${-cs} ${cs}`) .attr("x",d=>pm.left+d.ci*cs) .attr("y",d=>pm.top+d.ri*cs) .attr("width",cs) .attr("height",cs) .attr("fill",d=>d.v>0?interp_color(d.v):"none") .attr("stroke",d=>d.v>0?"none":d3.color("grey").brighter()); g_rect.append("title") .text(d=>`${metric.replace(" ","-")}(${dim.stk[d.ri].sn},${dim.stk[d.ci].sn})\n=${d.v.toFixed(3)}`); const g_sym_l = g.append("g").attr("transform",`translate(${pm.left/2},${pm.top+cs/2})`); const g_sym_l_path = g_sym_l.selectAll("path") .data(dim.stk) .join("path") .attr("transform",(d,i)=>`translate(${0},${cs*i})`) .attr("fill",d=>d.c) .attr("stroke","none") .attr("d",d=>d.sym()); g_sym_l_path .append("title") .text(d=>`(${d.sc}) ${d.sn}`); const g_sym_t = g.append("g").attr("transform",`translate(${pm.left+cs/2},${pm.top/2})`); const g_sym_t_path = g_sym_t.selectAll("path") .data(dim.stk) .join("path") .attr("transform",(d,i)=>`translate(${cs*i},${25})`) .attr("fill",d=>d.c) .attr("stroke","none") .attr("d",d=>d.sym()); g_sym_t_path .append("title") .text(d=>`(${d.sc}) ${d.sn}`); const g_title = g.append("text") .attr("text-anchor","middle").attr("dominant-baseline","hanging") .attr("x",gs/2) .attr("y",height-pm.bottom-4) .text(title); return {g:g,g_rect:g_rect,g_sym_l:g_sym_l,g_sym_t:g_sym_t,g_title:g_title}; } const g_corr_mat = make_g_matrix( mathjs.matrix(num.corr.map(r=>r.map(d=>d.v))), x=>d3.interpolateMagma(1-x), "0.원본 데이터", "correlation distance"); const fss = num.fss[ki]; const knn_corr = mathjs.zeros(fss.length,fss.length,'sparse'); fss.forEach((r,ri)=>{ r.nn.forEach((d,ci)=>{ knn_corr.set([ri,d.ci],d.v); }) }); const g_knn_corr_graph = make_g_flat_graph( knn_corr, x=>d3.interpolateMagma(1-x), `1. 추려낸 k(${presets.knn[ki]}) 최근접 그래프`, "correlation distance", true, ); const g_knn_corr_mat = make_g_matrix( knn_corr, x=>d3.interpolateMagma(1-x), `1. 추려낸 k(${presets.knn[ki]}) 최근접 데이터`, "correlation distance"); const fsm = mathjs.zeros(fss.length,fss.length,'sparse'); fss.forEach((r,ri)=>{ r.nn.forEach((d,ci)=>{ fsm.set([ri,d.ci],d.p); }) }); const g_fss_graph = make_g_flat_graph( fsm, d3.interpolateTurbo, `2. 개별 연결 강도로 표현된 그래프`, "membership strength", false ); const g_fss_mat = make_g_matrix( fsm, d3.interpolateTurbo, `2. 개별 연결 강도로 표현된 데이터`, "membership strength"); const fsu = num.fsu[ki]; const g_fsu_graph = make_g_flat_graph( fsu, d3.interpolateTurbo, `3. 합산 연결 강도로 표현된 그래프`, "membership strength", false ); const g_fsu_mat = make_g_matrix( fsu, d3.interpolateTurbo, `3. 합산 연결 강도로 표현된 데이터`, "membership strength"); function make_g_embedding(init_ep, interp_color, title, metric) { const ep = init_ep; const epoch = epochs[init_ep]; const x = d3.scaleLinear().domain(epoch.ext.x).range([pm.left,gs-pm.right]); const y = d3.scaleLinear().domain(epoch.ext.y).range([gs-pm.bottom,pm.top]); const embeds = d3.range(dim.stk.length).map(i=>({x:epochs.map(ep=>ep.embed[i][0]),y:epochs.map(ep=>ep.embed[i][1])})); const edges = num.edges[ki].map(d=>epochs.map(ep=>({v1:ep.embed[d.r],v2:ep.embed[d.c],w:d.w}))); const g = g_scroll.append("g"); g.node().appendChild( Legend(d3.scaleSequential([0.0, 1.0], interp_color),{ title: metric, width:gs, marginLeft:pm.left, marginRight:pm.right })); const g_graph = g.append("g"); const g_edge = g_graph.append("g").selectAll("path") .data(edges) .join("path") .attr("fill","none") .attr("stroke",d=>interp_color(d[ep].w)) .attr("d",d=>`M ${x(d[ep].v1[0])} ${y(d[ep].v1[1])} L ${x(d[ep].v2[0])} ${y(d[ep].v2[1])}`); const g_vertex = g_graph.append("g").selectAll("path") .data(embeds) .join("path") .attr("transform",(d,i)=>`translate(${x(d.x[ep])},${y(d.y[ep])})`) .attr("fill",(_,i)=>dim.stk[i].c) .attr("stroke","lightgrey") .attr("d",(_,i)=>dim.stk[i].sym()); g_vertex.append("title") .text((_,i)=>dim.stk[i].sn); const g_iter = g.append("text") .attr("text-anchor","end").attr("dominant-baseline","auto").attr("font-size",12) .attr("x",gs-pm.right).attr("y",pm.top) .text(`초기값`); const g_title = g.append("text") .attr("text-anchor","middle").attr("dominant-baseline","hanging") .attr("x",gs/2).attr("y",height-pm.bottom-4) .text(title); return {g:g,g_graph:g_graph,g_edge:g_edge,g_vertex:g_vertex,g_iter:g_iter,g_title:g_title}; } const interp_bw = d3.interpolateLab("white","black"); const g_init_embed = make_g_embedding(0,interp_bw, "4. 배치 초기화 by Spectral Embedding", "membership strength"); const g_embed = make_g_embedding(0,interp_bw, "5. 배치 최적화 by UMAP embedding optimization", "membership strength"); const pages = [g_corr_mat, g_knn_corr_graph, g_knn_corr_mat, g_fss_graph, g_fss_mat, g_fsu_graph, g_fsu_mat, g_init_embed, g_embed]; pages.forEach((g_page,pi)=>{g_page.g.attr("transform",`translate(${gs*pi},${0})`)}); const step_pages = [ [g_corr_mat], [g_knn_corr_graph,g_knn_corr_mat], [g_fss_graph,g_fss_mat], [g_fsu_graph,g_fsu_mat], [g_init_embed], [g_embed], ]; const step_page_range = step_pages.map(ps=>[pages.indexOf(ps[0]),pages.indexOf(ps[ps.length-1])+1]); return Object.assign(svg.node(),{ update: function(si) { const page_start_i = steps[si].i; const page_range = step_page_range[si]; g_scroll.transition().duration(anim_ms).attr("transform",`translate(${-gs*page_start_i},${0})`); pages.forEach((g,pi)=>{ g.g.attr("visibility",page_range[1]<=pi?"hidden":"visible"); }); step_pages[si].forEach(g=>{ g.g.interrupt().attr("opacity",0).transition().duration(anim_ms).attr("opacity",1); }); if(si==1 || si==2 || si==3) { const g_mat = step_pages[si][1]; const g_graph = step_pages[si][0]; g_mat.g_rect.interrupt().attr("opacity",0) .transition().delay(d=>anim_ms*(d.ri+1)).duration(anim_ms).attr("opacity",1); g_graph.g_grp.interrupt().attr("opacity",0) .transition().delay((_,ri)=>anim_ms*(ri+1)).duration(anim_ms).attr("opacity",1); g_graph.g_rhs.interrupt().attr("transform",d=>`translate(${gx(0)},${d.off*2})`) .transition().delay(d=>anim_ms*(d.ri+1)).duration(anim_ms) .attr("transform",d=>`translate(${gx(d.v)},${d.off*2})`); g_graph.g_edge.interrupt().attr("d",d=>`M ${gx(0)} ${d.off*2} H ${gx(0.01)}`) .transition().delay(d=>anim_ms*(d.ri+1)).duration(anim_ms) .attr("d",d=>`M ${gx(0)} ${d.off*2} H ${gx(d.v)}`); } else if(si==4) { let g_vertex = g_init_embed.g_vertex; let g_edge = g_init_embed.g_edge; g_vertex.interrupt().attr("opacity",0).transition().delay(anim_ms).duration(anim_ms).attr("opacity",1); g_edge.interrupt().attr("opacity",0).transition().delay(anim_ms*2).duration(anim_ms).attr("opacity",1); } else if(si==5) { let x = d3.scaleLinear().domain(epochs[0].ext.x).range([pm.left,gs-pm.right]); let y = d3.scaleLinear().domain(epochs[0].ext.y).range([gs-pm.bottom,pm.top]); let [g_vertex,g_edge,g_iter] = [g_embed.g_vertex,g_embed.g_edge,g_embed.g_iter]; g_vertex = g_vertex.interrupt().attr("transform",(d,i)=>`translate(${x(d.x[0])},${y(d.y[0])})`); g_edge = g_edge.interrupt().attr("d",d=>`M ${x(d[0].v1[0])} ${y(d[0].v1[1])} L ${x(d[0].v2[0])} ${y(d[0].v2[1])}`); g_iter = g_iter.interrupt().text("초기값"); d3.range(1,epochs.length).forEach(ep=>{ x = d3.scaleLinear().domain(epochs[ep].ext.x).range([pm.left,gs-pm.right]); y = d3.scaleLinear().domain(epochs[ep].ext.y).range([gs-pm.bottom,pm.top]); g_vertex = g_vertex.transition().delay(anim_ms).duration(anim_ms) .attr("transform",(d,i)=>`translate(${x(d.x[ep])},${y(d.y[ep])})`); g_edge = g_edge.transition().delay(anim_ms).duration(anim_ms) .attr("d",d=>`M ${x(d[ep].v1[0])} ${y(d[ep].v1[1])} L ${x(d[ep].v2[0])} ${y(d[ep].v2[1])}`); g_iter = g_iter.transition().delay(anim_ms).duration(anim_ms) .textTween(()=>t=>`#반복: ${d3.interpolateNumber(epochs[ep-1].ep,epochs[ep].ep)(t).toFixed(0)}`); }); } } }); }36:Taf6,num = { function add_index(m) { return m.map((r,ri)=>r.map((v,ci)=>({ri:ri,ci:ci,v:v}))); } function smooth_knn_dist(nn,rho,n_iter=64) { let [lo,hi,mid,k] = [0.0, Number.MAX_VALUE, 1.0, nn.length]; const target = Math.log2(k+1); for (const it of d3.range(n_iter) ) { const psum = d3.sum(nn.map(d=>Math.exp(-(d.v-rho)/mid))); if(Math.abs(psum-target) < 1e-5) break; if(psum > target) { hi = mid; mid = (lo+hi)/2.0; } else { lo = mid; mid = (hi == Number.MAX_VALUE) ? mid*2 : (lo+hi)/2.0; } } return mid; } function extract_fs_set(sorted, k) { const rows = sorted.map(r=>({nn:r.slice(0,k).map(d=>({ri:d.ri,ci:d.ci,v:d.v}))})); rows.forEach(r=>{ r.rho = r.nn[0].v; r.sigma = smooth_knn_dist(r.nn,r.rho); r.nn.forEach(d=>{ d.p = Math.exp(-(d.v-r.rho)/r.sigma); }); }); return rows; } function get_fuzzy_union_matrix(fss) { const n = fss.length; const fsm = mathjs.zeros(n,n,'sparse'); fss.forEach((r,ri)=>{ r.nn.forEach((d,ci)=>{ fsm.set([ri,d.ci],d.p); }) }); const fsm_t = mathjs.transpose(fsm); const fsu = mathjs.subtract(mathjs.add(fsm, fsm_t),mathjs.dotMultiply(fsm,fsm_t)); return fsu; } function spectral_embedding(graph) { const n = mathjs.size(graph)[0]; const sqrt_deg = mathjs.map(mathjs.squeeze(mathjs.sum(graph,0).toArray()),mathjs.sqrt); const I = mathjs.identity(n); const D = mathjs.diag(mathjs.map(sqrt_deg,x=>1/x)); const L = mathjs.subtract(I,mathjs.multiply(mathjs.multiply(D,graph),D)); const embedding = mathjs.transpose(mathjs.matrix(mathjs.eigs(L).eigenvectors.slice(1,3).map(d=>d.vector.toArray()))); const expasion = 10.0/mathjs.max(embedding); const scaled = embedding.map(x=>x*expasion); const [scaled_max, scaled_min] = [mathjs.max(scaled,0), mathjs.min(scaled,0)]; const normalized_10 = mathjs.dotMultiply(10,mathjs.dotDivide(mathjs.subtract(scaled,scaled_min),mathjs.subtract(scaled_max,scaled_min))); return normalized_10.toArray(); } const num_corr = add_index(raw_data.corr); const num_sorted = num_corr.map((r,ri)=>r.toSorted((a,b)=>d3.ascending(a.v,b.v)).filter(d=>d.ci!=ri)); const num_fss = presets.knn.map(k=>extract_fs_set(num_sorted,k)); const num_fsu = num_fss.map(fss=>get_fuzzy_union_matrix(fss)); const num_pairs = num_fsu.map(fsu=>{ const pairs = []; fsu.forEach((w,rc)=>{ pairs.push({r:rc[0],c:rc[1],w:w}); },true); return pairs; }); const num_edges = num_pairs.map(pairs=>pairs.filter(d=>d.rspectral_embedding(fsu)), pairs: num_pairs, edges: num_edges, }; return num; }37:Ta34,epochs = { const [a,b] = curves[params.min_dist_index].params; const embed = num.init[params.knn_index].map(d=>[...d]); const n_embed_history_samples = 5; const n_epochs = 500; const negative_sample_rate = 5; const gamma = 1.0; let alpha = 1.0; const alpha_decay_per_ep = alpha/n_epochs; const epochs_to_sample = new Set(d3.range(n_embed_history_samples) .map(i=>Math.floor((i+1)*n_epochs/n_embed_history_samples))); const embed_history = [ {ep:0,embed:embed.map(d=>[...d])} ]; function rdist(x,y) { return x.reduce((acc,cur,i)=>{ const diff = x[i]-y[i]; return acc + diff*diff; },0.0); } function clip(v) { return Math.max(-4.0,Math.min(4.0,v)); } const pairs = num.pairs[params.knn_index]; const weights = pairs.map(d=>d.w); const eps = mathjs.dotDivide(mathjs.max(weights),weights); // epochs_per_sample const epns = mathjs.dotDivide(eps,negative_sample_rate); // epochs_per_negative_sample const a_eps = [...eps]; // accumulated epochs per sample const a_epns = [...epns]; // accumulated epochs per negative sample for (const ep of d3.range(n_epochs)) { for (const [i,pair] of pairs.entries()) { if (a_eps[i] > ep) continue; // gradient adjustment per sample const [lhs,rhs] = [embed[pair.r],embed[pair.c]]; const dist_sq = rdist(lhs,rhs); const grad_coeff = dist_sq > 0.0 ? (-2.0*a*b*Math.pow(dist_sq,b-1.0))/(a*Math.pow(dist_sq,b)+1.0):0.0; lhs.forEach((_,d)=>{ const grad_d = clip(grad_coeff*(lhs[d]-rhs[d])); lhs[d] += grad_d*alpha; rhs[d] += -grad_d*alpha; }); a_eps[i] += eps[i]; // gradient adjustment for negative samples const pot = d3.range(embed.length).filter(ei=> ei!= pair.r & ei!=pair.c); const n_neg_samples = Math.floor((ep-a_epns[i])/epns[i]); d3.shuffle(pot); for(const neg_c of pot.slice(0,n_neg_samples)) { const rhs = embed[neg_c]; const dist_sq = rdist(lhs,rhs); const grad_coeff = dist_sq > 0.0 ? 2.0*gamma*b/((0.001+dist_sq)*(a*Math.pow(dist_sq,b)+1)) : 0.0; lhs.forEach((_,d)=>{ const grad_d = grad_coeff > 0.0 ? clip(grad_coeff*(lhs[d]-rhs[d])) : 0.0; lhs[d] += grad_d*alpha; }); } a_epns[i] += n_neg_samples*epns[i]; } if(epochs_to_sample.has(ep+1)) embed_history.push({ep:ep+1,embed:embed.map(d=>[...d])}); alpha = Math.max(alpha-alpha_decay_per_ep,1e-5); } embed_history.forEach((ep,i)=>{ ep.ext = {x:d3.extent(ep.embed.map(d=>d[0])),y:d3.extent(ep.embed.map(d=>d[1]))}; }); return embed_history; }38:[{"id":"bb090b8e0234178092b23548b045fbb47c80a105fb3a8356d4f87edab0e9ca024e8d0e61e8bec4dac8bec0d8a229d055b9a4f4548d552a8a1044f7a3d3cc6a60","href":"https://static.observableusercontent.com/files/bb090b8e0234178092b23548b045fbb47c80a105fb3a8356d4f87edab0e9ca024e8d0e61e8bec4dac8bec0d8a229d055b9a4f4548d552a8a1044f7a3d3cc6a60","name":"krx_correlation_6g_22s_2779d_20260429.json","type":"application/json","lastModified":1777458686967.524,"size":11089}] 31:{"id":"8a669b614cfb80cb","slug":"short-guide-to-ml-umap","title":"짧은 머신러닝 안내서: UMAP","thumbnail":"8f3ef2ad7f12433a256d479a904233781f27f796fdced1c3776a231f380797ec","access_level":"public","create_time":"$D2026-04-29T10:28:49.901Z","update_time":"$D2026-05-15T10:03:46.956Z","view_time":null,"first_public_time":"$D2026-05-14T22:40:05.947Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"50faa4da52a9ff9b","login":"humble-data-miner","name":"Humble Data Miner","avatar_url":"https://avatars-next.observableusercontent.com/3046b1bea86c5e825baea7c0c403d90ad9690b5396b11b5d02b3a0839fbcb946","tier":"starter_2024"},"version":1158,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"4e2fc41e06361cc3","login":"humble-data-miner","name":"Humble Data Miner","avatar_url":"https://avatars-next.observableusercontent.com/3046b1bea86c5e825baea7c0c403d90ad9690b5396b11b5d02b3a0839fbcb946","tier":null}],"members":[{"type":"individual","id":"4e2fc41e06361cc3","login":"humble-data-miner","name":"Humble Data Miner","avatar_url":"https://avatars-next.observableusercontent.com/3046b1bea86c5e825baea7c0c403d90ad9690b5396b11b5d02b3a0839fbcb946","tier":null,"role":"editor"},{"type":"team","id":"50faa4da52a9ff9b","login":"humble-data-miner","name":"Humble Data Miner","avatar_url":"https://avatars-next.observableusercontent.com/3046b1bea86c5e825baea7c0c403d90ad9690b5396b11b5d02b3a0839fbcb946","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":{"id":"b886df04acb387aa","slug":"short-guide-to-machine-learning","title":"Short Guide to Machine Learning","owner":{"id":"50faa4da52a9ff9b","login":"humble-data-miner","name":"Humble Data Miner"}},"type":"notebook","body":{"title":"짧은 머신러닝 안내서: UMAP","cells":[{"id":0,"value":"# 짧은 머신러닝 안내서: UMAP\n## 사람의 인지 방식과 닮아 결과가 보기 편한 차원 축소 알고리즘","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":691,"value":"---\n## UMAP?\n* **U**niform **M**anifold **A**pproximation and **P**rojection\n* t-SNE와 유사한 차원 축소 알고리즘\n * 고(N)차원 데이터를 주로 평면(2차원)상에 표현\n* [UMAP official website](https://umap-learn.readthedocs.io/)\n\n

\n## t-SNE의 대체재?\n* 공개적으로 많은 이론적 배경이 있고 다양한 장점이 있음을 강조\n* 아래 예제 데이터 상의 UMAP 동작을 먼저 살펴보고 대체 가능성을 판단하려함","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":38,"value":"---\n## 데이터셋\n* 가격 일변동 데이터: KRX ${dim.stk.length} 종목 × 2779 거래일\n${Inputs.table(dim.stk,{ \n header:{ sc:\"종목코드\", sn:\"종목이름\", gn:\"종목그룹\"},\n columns:[ \"sc\", \"sn\", \"gn\" ],\n format:{\n sn:(d,i)=>html`\n \n ${d}`,\n gn:(d,i)=>html`${d}` },\n width:{sc:60,gn:120},\n maxWidth:430,\n rows:dim.stk.length+2,\n })}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":813,"value":"$33","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1065,"value":"

\n## UMAP 알고리즘 동작 예시\n* 사용자 입력 파라메터\n * \\# nearest neighbors: 한 종목당 고려할 근접 데이터 수, knn의 k값\n * 문서에서 제안하는 파라메터 값 범위는 [5 .. 50]\n * 낮을수록 군집들이 분리되고 높을수록 군집들이 가까워지는 경향\n * min dist: UMAP embedding 과정에만 사용하는 서로 밀어내는 힘 조절\n * 문서에서 제안하는 파라메터 값 범위는 [0.001 .. 0.5]\n * 낮을수록 군집내에서 밀집 높을수록 군집내에서 산개하는 경향","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":244,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":546,"value":"viewof step = {\n const sl_step = d3_slider.sliderBottom()\n .min(0).max(presets.step.length-1).step(1)\n .width(width-100)\n .value(0)\n .ticks(presets.knn.length)\n .tickFormat(i=>steps[i].l)\n .fill(\"black\")\n .on(\"onchange\", () => svg.dispatch(\"input\"));\n\n const svg = d3.create(\"svg\")\n .attr(\"viewBox\", [0, 0, width, 62]);\n\n const g_sl_step = svg.append(\"g\").attr('transform', `translate(${40},${24})`);\n g_sl_step.call(sl_step);\n g_sl_step.append(\"text\").attr(\"transform\",`translate(${width/2-50},${-8})`).attr(\"text-anchor\",\"middle\").text(\"진행 과정 슬라이더\");\n return Object.defineProperty(\n svg.node(), \n \"value\",\n {get: () => sl_step.value()}\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1089,"value":"---\n## UMAP 특성 요약\n\n* UMAP의 핵심 아이디어 **\"k근접점으로 한정된 로컬 데이터 공간내 연결강도 재해석\"**\n * k근접점 밖의 다른점들을 배치 계산에서 완전히 제외 → 군집간에 명확한 분리\n * 군집내에서 데이터간의 배치거리와 실제거리상의 괴리 → 배치점간 거리 부정확\n * 이상치(Outlier)도 k근접점 로컬 데이터 공간을 생성 → 어쨌든 군집에 연결됨\n\n* UMAP의 인간적(?) 특성 **\"사람이 봤을 때 인지적으로 편안한 배치 결과\"**\n * 사람의 제한된 탐색 범위 → UMAP의 한정된 k개 근접점 탐색\n * 사람의 순서 중심 탐색 → UMAP의 로컬 공간내 순서기반 연결강도 계산\n\n* 쓰임새에 대한 개인적인 의견 **\"Clustered data visualization 알고리즘\"**\n * 비슷한 특성을 갖는 군집을 찾는데 유용\n * 군집내 개별 데이터간 거리를 비교하기에는 부정확\n * 이상치(Outlier)가 별로 없는 데이터에 효과적","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1119,"value":"---\n## Reference\n* [UMAP documentation](https://umap-learn.readthedocs.io/)\n* [UMAP paper](https://arxiv.org/abs/1802.03426)\n* [UMAP github](https://github.com/lmcinnes/umap)\n* [scipy.spatial.distance.correlation](https://docs.scipy.org/doc/scipy/reference/generated/scipy.spatial.distance.correlation.html)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1157,"value":"---\n## 부연설명 동영상\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":5,"value":"---\n## Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":604,"value":"anim_ms = 500","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":612,"value":"input_data = ({pi:0})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"raw_data = FileAttachment(\"krx_correlation_6g_22s_2779d_20260429.json\").json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"dim = {\n const data = JSON.parse(JSON.stringify(raw_data.stk));\n const gc_list = data.reduce((acc,cur)=>{ if(acc.indexOf(cur.gc)<0) { acc.push(cur.gc); } return acc; },[])\n const gc_color_map = Object.fromEntries(gc_list.map((gc,i)=>[gc,d3.color(d3.schemeCategory10[i])]));\n let grp_cnt = Object.fromEntries(gc_list.map(gc=>[gc,0]));\n data.forEach(d=>{\n d.c = gc_color_map[d.gc];\n d.sym = d3.symbol(d3.symbolsFill[grp_cnt[d.gc]]);\n grp_cnt[d.gc] += 1;\n });\n return {stk:data,gclist:gc_list};\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":48,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":92,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":117,"value":"mathjs = require('mathjs@15.2.0/lib/browser/math.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":182,"value":"ml_curve_fit = import('https://unpkg.com/ml-levenberg-marquardt@5.0.1/lib/index.js?module')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":189,"value":"LM = ml_curve_fit.levenbergMarquardt","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":247,"value":"d3_slider = require('d3-simple-slider@2.0.0/dist/d3-simple-slider.min.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":366,"value":"import {Legend, Swatches} from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"presets = {\n return {\n knn: d3.range(2,6+1,1),\n min_dist: d3.range(0.1,1.0,0.1),\n step: d3.range(6),\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":569,"value":"steps = [\n {l:\"0.원본 거리 데이터\",i:0},\n {l:\"1.k근접 필터링 (비대칭)\",i:0},\n {l:\"2.개별 연결강도 계산 (비대칭)\",i:2},\n {l:\"3.연결강도 합산 (대칭)\",i:4},\n {l:\"4.배치 초기화 by Spectral Embedding\",i:6},\n {l:\"5.배치 최적화 by UMAP\",i:6},\n];","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":195,"value":"curves = {\n const spread = 1.0;\n const min_dists = presets.min_dist;\n function smooth_curve([a, b]) {\n return (x) => 1.0 / (1.0 + a * x**(2*b));\n }\n \n const xv = d3.range(0,spread*3+1e-5,spread/100);\n return min_dists.map((min_dist,i)=>{\n const yv = xv.map(x=>x