mongoData.filter(d=>d.lastPlaceOfService)
d3.nest().key(d=>d.lastPlaceOfService).entries(mongoData).filter(d=>d.values.length==5)
cityArmyStr = JSON.stringify(cityArmy)
armyPlacesStr = JSON.stringify(armyPlaces)
armyPlaces = getSimilarArmySoldiers(placePersons)
function getSimilarArmySoldiers(persons){
const nested = d3.nest().key(d=>d.lastPlaceOfService).entries(persons)
.filter(d=>d.key)
// sort by death date
nested.forEach(d=>{
d.values.sort((a,b)=>{
const splitted = [a,b].map(d=>d.deathDate.split('.'));
const built = splitted.map(s=>s[2]+'.'+s[1]+'.'+s[0]);
// resolve(built)
return built[0]>built[1]?1:-1
}) ;
d.values = d.values.filter((d,i,arr)=>{if(i>0 && arr[i].burialLocation==arr[i-1].burialLocation) return false;return true;})
})
const result = nested
.sort((a,b)=>a.values.length<b.values.length?1:-1)
result.forEach(d=>{
d.values = d.values.map(v=>{
return {
date:v.deathJsDate ,
burialLocation:v.burialLocation
}
})
})
return result;
}
placePersons = getPlacePersons()
cityArmy = d3.nest().key(d=>d.place)
.entries(placePersons)
.filter(d=>+d.key)
.sort((a,b)=>a.values.length<b.values.length?1:-1)
.map(d=>({city:d.key,army:getSimilarArmySoldiers(d.values).slice(0,10).map(d=>d.key)}))
function getPlacePersons(place){
return new Promise((resolve,reject)=>{
resolve(mongoData
.map(d=>{
const splitted = d.deathDate.split('.');
return Object.assign(d,{deathJsDate:new Date(splitted[2],splitted[1],splitted[0])})
})
.filter(obj=>{const d= obj.deathJsDate; return d instanceof Date && !isNaN(d);})
.filter(d=>d.burialLocation!='1')
.filter(d=>d.burialLocation!='')
.filter(d=>+d.burialLocation)
.filter(d=>d.deathDate)
.filter(d=>d.deathDate.split('.')[0])
) ;
})
}
function getAllServiceSoldiers(arrOfServicePlaces){
return new Promise((resolve,reject)=>{
resolve(mongoData.filter(d=>arrOfServicePlaces.indexOf(d.lastPlaceOfService)!=-1))
})
}
mapGeoData = driveData.regions.elements.filter(d=>d.lat && d.lng)
.map(d=>{
const result = {};
result.index = d.index;
result.population = d.records ;
result.cityGeo = d.geo;
result.long = d.lng;
result.lat = d.lat;
return result;
})
.filter(d=>d.cityGeo!="ორჯონიკიძე")
.filter(d=>d.cityGeo!="ბორჩალო")
.filter(d=>d.cityGeo!="წითელწყარო")
.filter(d=>d.cityGeo!="ზნაური")
.filter(d=>d.cityGeo!="ქვემო სვანეთი")
.filter(d=>d.cityGeo!="თუშეთი")
.filter(d=>d.cityGeo!="ყაზბეგი")
lastNamesMatched = d3.csv('https://raw.githubusercontent.com/bumbeishvili/georgian-names/master/RusEnGeoMatch/lastNamesRusGeoEng.csv')
lastNamesMatchedObj = {
const result = {};
lastNamesMatched.forEach(function(d){
result[d.rus]=d.matched
})
return result;
}
lastNamesToCompare = lastNamesTop
//.filter((d,i)=>i<3000)
rusEnLastNames = lastnamesNested
.map(d=>({rus:d.key,en:d.values[0].lname
.toLowerCase()
}))
//.filter((d,i)=>i<3)
rusEnLastNames
downloadButton(rusEnLastNames)
downloadButton(lastNamesToCompare)
lastNamesToCompare
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
rusEnNames = namesNested.map(d=>({rus:d.key,en:d.values[0].fname}))//.filter((d,i)=>i<10000)
namesToCompare = namesTop
.filter((d,i)=>i<10000)
.filter((d)=>femaleNamesTop.indexOf(d.nameGe)==-1)
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
rusEnNamesMatched = rusEnNames.map(d=>{
return;
/*
let dist = Infinity;
let matchedName = '';
for(var i=0;i<namesToCompare.length;i++){
const name = namesToCompare[i];
const distance = getEditDistance(d.en.toLowerCase(),name.nameEn);
if(distance<dist){
dist = distance;
matchedName = name.nameGe;
}
if(dist==0){
break;
}
}
return Object.assign(d,{matched:matchedName})
*/
})
fittedData = mongoData.map(d=>{
const rec = {};
rec.id = d.id;
rec.name = d.nameGe;
rec.lastName = d.lastNameGe;
rec.birthDate = d.birthDate;
rec.lastPlaceOfService = d.lastPlaceOfService;
rec.place =driveDataIndexObj.regions[d.place]&&driveDataIndexObj.regions[d.place].geo||d.place;
rec.rank = driveDataIndexObj.rank[d.rank]&&driveDataIndexObj.rank[d.rank].geo||d.rank;
rec.burialLocation = driveDataIndexObj.burialLocation[d.burialLocation]&&driveDataIndexObj.burialLocation[d.burialLocation].geo||'';
rec.deathDate = d.deathDate;
rec.deathReason = driveDataIndexObj.deathReason[d.deathReason]&&driveDataIndexObj.deathReason[d.deathReason].geo||d.deathReason==1?"":d.deathReason;
rec.url = d.url;
return rec;
})
function getSearchResults(name,lastname,cityReg){
return mongoData
.filter(d=>lastname==null || lastname=='' || lastname==d.lastName)
.filter(d=>name==null || name=='' || d.name==name)
.filter(d=>cityReg==null || cityReg==''|| cityReg==d.place)
.filter((d,i)=>i<30)
}
function searchName(name){
const nameResult = Object.keys(driveDataObj.names)
.filter(k=>driveDataObj.names[k].geo==name)[0];
if(nameResult){
return driveDataObj.names[nameResult].index
}
}
function searchLastName(lastName){
const nameResult = Object.keys(driveDataObj.lastnames)
.filter(k=>driveDataObj.lastnames[k].geo==lastName)[0];
if(nameResult){
return driveDataObj.lastnames[nameResult].index
}
}
downloadButton(mongoData)
lastNamesMatchedObj
mongoData.filter(d=>d.lastNameEn.indexOf('feradze')!=-1)
mongoData = withEngNames.map(d=>{
const id = d.id;
const url = d.url;
let nameGe = manualNamesMatchedObj[d['first name']];
let nameEn= d['fname'];
if(nameEn== ""){
nameGe="";
}
let lastNameGe = lastNamesMatchedObj[d['last name']];
let lastNameEn=d['lname'];
let lastNameRus=d['last name'];
if(lastNameRus== "Гелашвили"){
lastNameGe="გელაშვილი";
lastNameEn='gelashvili';
}
const birthDate = d.birth_date;
const lastPlaceOfService = d['last place of service']
const place = driveDataObj.regions[getRegion(d)]? +driveDataObj.regions[getRegion(d)].index : d['date/place of recruitment'];
const rank = driveDataObj.rank[d['military rank']]? +driveDataObj.rank[d['military rank']].index : d['military rank'];
const deathDate = d["death date"]
const burialLocation = driveDataObj.burialLocation[getBurialLocation(d)]? +driveDataObj.burialLocation[getBurialLocation(d)].index : d['initial burial location'];
const deathReason = driveDataObj.deathReason[d['death reason']]? +driveDataObj.deathReason[d['death reason']].index : d['death reason']
//deathReason
return {
id,
nameGe,
nameEn:nameEn.toLowerCase(),
lastNameGe,
lastNameEn:lastNameEn.toLowerCase(),
birthDate,
lastPlaceOfService,
place,
rank,
burialLocation,
deathDate,
deathReason,
url
}
})
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
driveData = load("https://docs.google.com/spreadsheets/d/123zrq8tW1BbXKg27G7LZBZ1HonEMpDeDsCpToemabqY/edit#gid=1055102699",true)
driveDataObj = {
const result = {};
const keys = Object.keys(driveData);
keys.forEach(d=>{
result[d]={};
const arr = driveData[d];
arr.elements.forEach(item=>{
result[d][item.rus]=item;
})
})
return result;
}
militaryRanksNested = d3.nest()
.key(d=>d['military rank'])
.entries(withEngNames)
.sort((a,b)=>a.values.length<b.values.length?1:-1)
//html`<table>${militaryRanksNested.map(d=>[d.key,d.values.length]).map(table).join('')}</table>`
deathReasonNested = d3.nest()
.key(d=>d['death reason'])
.entries(withEngNames)
.sort((a,b)=>a.values.length<b.values.length?1:-1)
//html`<table>${deathReasonNested.map(d=>[d.key,d.values.length]).map(table).join('')}</table>`
md`## Burial location`
getBurialLocation=d=>d['initial burial location'].split(",")[0]
burialsRegions = d3.nest()
.key(getBurialLocation)
.entries(withEngNames)
.sort((a,b)=>a.values.length<b.values.length?1:-1)
.filter(d=>d.values.length>5)
.map(d=>[d.key,d.values.length])
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
//html`<table>${burialsRegions.map(table).join('')}</table>`
//html`<table>${namesFiltered.map(table).join('')}</table>`
//html`<table>${lastNamesFiltered.map(table).join('')}</table>`
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
function getRegion(d){
var arr = d['date/place of recruitment']
.split(' ')
;
var result = arr[arr.length-1];
if(result=="р-н"){
result = arr[arr.length-2]
}
if(result=="г." || result=="ССР"){
result= arr[0]
}
if(result=="г." || result=="РВК"){
result= arr[arr.length-2]
}
if(result=="АССР" ){
result= arr[arr.length-1]
}
if(result.match(/.[_\d]/g)){
result= arr[arr.length-4]
}
if(result=="АССР"){
result= arr[1]
}
return result
}
function getGeo(result){
return driveDataObj.oldRegions[result] && driveDataObj.oldRegions[result].geo||result;
}
function getLat(result){
return driveDataObj.oldRegions[result] && driveDataObj.oldRegions[result].lat||'';
}
regionCitiesNested = d3.nest()
.key(d=> getGeo(getRegion(d)))
.entries(withEngNames)
.sort((a,b)=>a.values.length<b.values.length?1:-1)
regionCityFiltered = regionCitiesNested
.filter((d,i)=>i<94)
.map(d=>[getRegion(d.values[0]),d.key,d.values.length,getLat(getRegion(d.values[0])),getLng(getRegion(d.values[0]))])
html`<table>${regionCityFiltered.map(table).join('')}</table>`
regionCitiesNested
"__.__.1942 Сагареджинский РВК, Грузинская ССР".match(/.[_\d]/g)
"11.09.1941 Тианетский РВК, Грузинская ССР, Тианетский р-н".match(/.[_\d]/g)
"Ткибульский".match(/.[_\d]/g)
getMinDistanceCity = function getMinDistanceCity(cityName){
var minDist = Infinity;
var obj = null;
coords.forEach((d,i)=>{
var cDist = distance(d.nameEn,cityName);
if(cDist<minDist){
minDist = cDist;
obj = d;
}
})
return {obj:obj,distance:minDist};
}
posMappings = {
return {
"Тбилиси":'Tbilisi',
"Кутаиси":'Kutaisi',
"Поти":'Poti',
"Батуми":'Batumi',
"Сухуми":'Sukhum',
}
}
regionsMapped = regionsNested.map(d=>{
var corrCity = getMinDistanceCity(d.key);
return {
old:d.key,
population:d.values.length,
corrCity:corrCity.obj.nameEn,
distance:corrCity.distance,
length:d.key.length,
lat:corrCity.obj.lat,
long:corrCity.obj.long
}
})
.filter(d=>d.distance<6 || d.corrCity == "Chkhorotsku")
.filter(d=>d.distance!=5)
.filter(d=>d.distance!=4|| d.corrCity == "Tetritskaro")
.filter(d=>d.distance!=3)
//.filter(d=>d.old != "Suhumi" && d.old!="Hobi")
names = data.map(d=>{
var obj = {
id:d.record_id,
fname:conv.transform(d['first name']),
lname:conv.transform(d['last name']),
middle:conv.transform(d['middle name'])
};
return Object.assign({},obj,d)
})
function EncodingConverter() {
var geoToLatinBinding = [
'a', 'b', 'g', 'd', 'e', 'v', 'z', 'T', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'J', 'r', 's', 't', 'u', 'f', 'q', 'R', 'k', 'S', 'ch', 'c', 'dz', 'w', 'W', 'x', 'j', 'h'
];
var latinToGeoBinding = [
'A', 'B', 'ჩ', 'D', 'E', 'F', 'G', 'H', 'I', 'ჟ', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'ღ', 'შ', 'თ', 'U', 'V', 'ჭ', 'X', 'Y', 'ძ', '[', '\\', ']', '^', '_', '`', 'ა', 'ბ', 'ც', 'დ', 'ე', 'ფ', 'გ', 'ჰ', 'ი', 'ჯ', 'კ', 'ლ', 'მ', 'ნ', 'ო', 'პ', 'ქ', 'რ', 'ს', 'ტ', 'უ', 'ვ', 'წ', 'ხ', 'ყ', 'ზ'
];
this.toLatin = function(geoWord) {
return convert(geoWord, geoToLatinBinding, 'ა', 'ჰ', 4304);
}
this.toGeorgian = function(latinWord) {
return convert(latinWord, latinToGeoBinding, 'A', 'z', 65);
}
function convert(word, binding, min, max, charNum) {
var buffer = []
var i = 0;
word.split('').forEach(function(c) {
if (c >= min && c <= max) {
buffer[i++] = binding[c.charCodeAt(0) - charNum];
} else {
buffer[i++] = c;
}
});
return buffer.join('');
}
}
c = new EncodingConverter()
distance = function distance(a, b){
if(a.length == 0) return b.length;
if(b.length == 0) return a.length;
var matrix = [];
// increment along the first column of each row
var i;
for(i = 0; i <= b.length; i++){
matrix[i] = [i];
}
// increment each column in the first row
var j;
for(j = 0; j <= a.length; j++){
matrix[0][j] = j;
}
// Fill in the rest of the matrix
for(i = 1; i <= b.length; i++){
for(j = 1; j <= a.length; j++){
if(b.charAt(i-1) == a.charAt(j-1)){
matrix[i][j] = matrix[i-1][j-1];
} else {
matrix[i][j] = Math.min(matrix[i-1][j-1] + 1, // substitution
Math.min(matrix[i][j-1] + 1, // insertion
matrix[i-1][j] + 1)); // deletion
}
}
}
return matrix[b.length][a.length];
}
dataPart1 = d3.csv('https://raw.githubusercontent.com/bumbeishvili/data-house/f1662d81f308ab66921bb8b079d6890243d39eed/hackviz/datap1.csv')
dataPart2 = d3.csv('https://raw.githubusercontent.com/bumbeishvili/data-house/master/hackviz/datap2.csv')
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
table('test')
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
function table(item){
if(typeof item=="object"){
return '<tr>'+item.map(d=>`<td>${d}</td>`).join('')+'</tr>'
}
return `<tr><td>${item}</td></tr>`
}
a = ({
"Ё": "YO",
"Й": "I",
"Ц": "TS",
"У": "U",
"К": "K",
"Е": "E",
"Н": "N",
"Г": "G",
"Ш": "SH",
"Щ": "SCH",
"З": "Z",
"Х": "Kh",
"Ъ": "'",
"ё": "yo",
"й": "i",
"ц": "ts",
"у": "u",
"к": "k",
"е": "e",
"н": "n",
"г": "g",
"ш": "sh",
"щ": "sch",
"з": "z",
"х": "kh",
"ъ": "'",
"Ф": "F",
"Ы": "I",
"В": "V",
"А": "a",
"П": "P",
"Р": "R",
"О": "O",
"Л": "L",
"Д": "D",
"Ж": "ZH",
"Э": "E",
"ф": "f",
"ы": "i",
"в": "v",
"а": "a",
"п": "p",
"р": "r",
"о": "o",
"л": "l",
"д": "d",
"ж": "zh",
"э": "e",
"Я": "Ia",
"Ч": "CH",
"С": "S",
"М": "M",
"И": "I",
"Т": "T",
"Ь": "'",
"Б": "B",
"Ю": "IU",
"я": "ia",
"ч": "ch",
"с": "s",
"м": "m",
"и": "i",
"т": "t",
"ь": "'",
"б": "b",
"ю": "iu"
})
function transliterate(word){
return word.split('').map(function (char) {
return a[char] || char;
}).join("");
}
cirrilic = require('https://bundle.run/cyrillic-to-translit-js@2.1.0')
conv = ({transform:transliterate})
import {requireFromGithub,load} from '@bumbeishvili/fetcher'
import {downloadButton} from "@bumbeishvili/utils"
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.