dataShotSequences = convertPointsToShots(dataPointsClean)
// cell to debug rally decoding errors
// {
// const match = 101330;
// const point = 42;
// let i = 0;
// while (dataPoints[i].matchId != match) {
// i += 1;
// }
// return {
// i: i + point - 1,
// rally1: dataPoints[i+point-1].rally1,
// rally2: dataPoints[i+point-1].rally2,
// pointData: dataPoints[i+point-1],
// rawData: dataRaw[i+point-1]
// };
// }
convertPointsToShots = function(dataPointsClean) {
let data = [];
let currentMatch = -1;
let currentSetsPl1, currentSetsPl2, currentGamesPl1, currentGamesPl2, currentPointsPl1, currentPointsPl2;
let point, currentPoint, rally, pos, event, prevEvent, isPlayer1Shot, shouldSavePoint;
for (let p of dataPointsClean) {
shouldSavePoint = true;
// handle switching to plays from the next match
if (p.matchId == currentMatch) {
currentPoint += 1;
} else {
currentMatch = p.matchId;
currentPoint = 1;
currentSetsPl1 = 0;
currentSetsPl2 = 0;
currentGamesPl1 = 0;
currentGamesPl2 = 0;
currentPointsPl1 = 0;
currentPointsPl2 = 0;
}
// seed the sequence (point)
point = {
matchId: p.matchId,
pointNumber: currentPoint,
setsPlayer1: currentSetsPl1,
setsPlayer2: currentSetsPl2,
gamesPlayer1: currentGamesPl1,
gamesPlayer2: currentGamesPl2,
pointsPlayer1: currentPointsPl1,
pointsPlayer2: currentPointsPl2,
tiebreakPointNumber: p.tiebreakPointNumber,
isPlayer1Serving: !!p.isPlayer1Serving,
isPlayer1Winner: !!p.isPlayer1Winner,
isGameWinner: !!p.isGameWinner,
isSetWinner: !!p.isSetWinner,
//rally1: p.rally1,
//rally2: p.rally2,
events: []
};
// decode rally1+rally2 into events (shots)
if ((typeof p.rally1 !== 'string') || (p.rally1.length == 0)) {
//return 'missing rally1';
return rally + ' - ' + pos + ' - missing_rally1 - ' + p.matchId + ' - ' + currentPoint;
}
// save penalties and points with missing shot data
if (['S', 'R', 'P', 'Q'].includes(p.rally1[0])) {
if ((p.rally1.length > 1) || p.rally2) {
//return 'wrong entry for penalties or points with missing shot data';
return rally + ' - ' + pos + ' - penalties_no_data - ' + p.matchId + ' - ' + currentPoint;
}
switch (p.rally1[0]) {
case 'S':
point.events.push({ _eventName: 'server_point' });
shouldSavePoint = false;
break;
case 'R':
point.events.push({ _eventName: 'returner_point' });
shouldSavePoint = false;
break;
case 'P':
point.events.push({ _eventName: 'server_penalty' });
shouldSavePoint = false;
break;
case 'Q':
point.events.push({ _eventName: 'returner_penalty' });
shouldSavePoint = false;
break;
}
} else {
rally = p.rally1;
isPlayer1Shot = !!p.isPlayer1Serving;
pos = 0;
// capture 1st serve event
event = readServe(p);
if (typeof event === 'object') {
point.events.push(event);
} else {
return event;
}
// if 1st serve resulted in a fault - go to 2nd serve
if (event._eventName == 'serve_fault') {
rally = p.rally2;
pos = 0;
// capture 2nd serve event
//console.log([p.matchId, currentPoint]);
event = readServe(p);
if (typeof event === 'object') {
point.events.push(event);
} else {
return event;
}
}
// if 2nd serve resulted in a fault - double fault; otherwise, capture rally
if (event._eventName != 'serve_fault') {
// capture serve win outcome
if (['*', '#'].includes(rally[pos])) {
point.events.push({
_eventName: 'ace',
isUnreturnable: (rally[pos] == '#')
});
} else {
// capture rally shots
while ((pos < rally.length) && !['*', 'n', 'w', 'd', 'x', '!', 'C', 'e', '@', '#'].includes(rally[pos])) {
isPlayer1Shot = !isPlayer1Shot;
event = readShot(p);
if (typeof event === 'object') {
point.events.push(event);
} else {
return event;
}
}
// capture rally result
event = {
isPlayer1: !!isPlayer1Shot
};
if (['*', 'n', 'w', 'd', 'x', '!', 'C', 'e'].includes(rally[pos])) {
switch (rally[pos]) {
case '*':
event._eventName = 'winner';
break;
case 'n':
event.errorType = 'net';
break;
case 'w':
event.errorType = 'wide';
break;
case 'd':
event.errorType = 'deep';
break;
case 'x':
event.errorType = 'wide_deep';
break;
case '!':
event.errorType = 'shank';
event.isShank = true;
break;
case 'C':
//event._eventName = 'lost_challenge';
event._eventName = 'winner'; // lumping together for simplicity
event.hasLostChallenge = true;
break;
case 'e':
event.errorType = 'unknown';
shouldSavePoint = false;
break;
}
pos += 1;
}
if (pos < rally.length) {
// capture if error was forced or unforced
switch (rally[pos]) {
case '@':
event._eventName = 'error_unforced';
break;
case '#':
event._eventName = 'error_forced';
break;
}
}
if (event.hasOwnProperty('_eventName')) {
point.events.push(event);
} else if (event.hasOwnProperty('errorType')) {
event._eventName = 'error_unknown';
point.events.push(event);
} else {
shouldSavePoint = false;
//return 'wrong rally result';
//return rally + ' - ' + pos + ' - rally_result - ' + p.matchId + ' - ' + currentPoint;
}
}
}
}
// save point sequence
if (shouldSavePoint) {
data.push(point);
}
// update sets, games and points for the next point
if (p.isSetWinner) {
if (p.isPlayer1Winner) {
currentSetsPl1 += 1;
} else {
currentSetsPl2 += 1;
}
currentGamesPl1 = 0;
currentGamesPl2 = 0;
currentPointsPl1 = 0;
currentPointsPl2 = 0;
} else if (p.isGameWinner) {
if (p.isPlayer1Winner) {
currentGamesPl1 += 1;
} else {
currentGamesPl2 += 1;
}
currentPointsPl1 = 0;
currentPointsPl2 = 0;
} else {
if (p.isPlayer1Winner) {
currentPointsPl1 += 1;
} else {
currentPointsPl2 += 1;
}
}
}
function readServe(p) {
// seed the event
let ev = {
isPlayer1: !!isPlayer1Shot
};
// skip let serves
while (rally[pos] == 'c') {
pos += 1;
}
let posServeStart = pos;
while (['4', '5', '6', '0', '+', '^', ';', 'n', 'w', 'd', 'x', 'g', '!', 'V', 'e'].includes(rally[pos])) {
switch (rally[pos]) {
case '4':
ev._eventName = 'serve_wide';
break;
case '5':
ev._eventName = 'serve_body';
break;
case '6':
ev._eventName = 'serve_middle';
break;
case '0':
ev._eventName = 'serve_unknown';
shouldSavePoint = false;
break;
case '+':
ev.courtPosition = 'approach';
break;
case '^':
ev.isDropVolley = true;
break;
case ';':
ev.hasClippedNet = true;
break;
case 'n':
ev.faultType = 'net';
break;
case 'w':
ev.faultType = 'wide';
break;
case 'd':
ev.faultType = 'deep';
break;
case 'x':
ev.faultType = 'wide-deep';
break;
case 'g':
ev.faultType = 'foot';
break;
case '!':
ev.faultType = 'shank';
ev.isShank = true;
break;
case 'V':
ev.faultType = 'time';
break;
case 'e':
ev.faultType = 'unknown';
break;
}
pos += 1;
}
if (pos == posServeStart) {
//return 'cannot read serve';
return rally + ' - ' + pos + ' - serve - ' + p.matchId + ' - ' + currentPoint;
}
if (ev.hasOwnProperty('faultType')) {
ev._eventName = 'serve_fault';
}
return ev;
}
function readShot(p) {
// seed the event
let ev = {
isPlayer1: !!isPlayer1Shot
};
// capture shot type
switch (rally[pos]) {
case 'f':
ev._eventName = 'forehand';
break;
case 'b':
ev._eventName = 'backhand';
break;
case 'r':
ev._eventName = 'f_slice';
break;
case 's':
ev._eventName = 'b_slice';
break;
case 'v':
ev._eventName = 'f_volley';
break;
case 'z':
ev._eventName = 'b_volley';
break;
case 'o':
ev._eventName = 'f_smash';
break;
case 'p':
ev._eventName = 'b_smash';
break;
case 'u':
ev._eventName = 'f_drop';
break;
case 'y':
ev._eventName = 'b_drop';
break;
case 'l':
ev._eventName = 'f_lob';
break;
case 'm':
ev._eventName = 'b_lob';
break;
case 'h':
ev._eventName = 'f_half_volley';
break;
case 'i':
ev._eventName = 'b_half_volley';
break;
case 'j':
ev._eventName = 'f_swing_volley';
break;
case 'k':
ev._eventName = 'b_swing_volley';
break;
case 't':
ev._eventName = 'trick';
break;
case 'q':
ev._eventName = 'unknown';
shouldSavePoint = false;
break;
default:
//return 'cannot read shot type';
return rally + ' - ' + pos + ' - shot_type - ' + p.matchId + ' - ' + currentPoint;
}
pos += 1;
// capture court position, special situations, direction of the shot and return depth
while (['+', '-', '=', ';', '^', '!', '1', '2', '3', '7', '8', '9', '0'].includes(rally[pos])) {
switch (rally[pos]) {
case '+':
ev.courtPosition = 'approach';
break;
case '-':
ev.courtPosition = 'net';
break;
case '=':
ev.courtPosition = 'baseline';
break;
case ';':
ev.hasClippedNet = true;
break;
case '^':
ev.isDropVolley = true;
break;
case '!':
ev.isShank = true;
break;
case '1':
ev.shotDirection = 'forehand';
break;
case '2':
ev.shotDirection = 'middle';
break;
case '3':
ev.shotDirection = 'backhand';
break;
case '7':
ev.returnDepth = 'short';
break;
case '8':
ev.returnDepth = 'middle';
break;
case '9':
ev.returnDepth = 'deep';
break;
case '0':
if (ev.hasOwnProperty('shotDirection')) {
ev.returnDepth = 'unknown';
} else {
ev.shotDirection = 'unknown';
}
break;
}
pos += 1;
}
return ev;
}
return data;
}
// rally data is entered manually and contains some errors - here we fix biggest errors
dataPointsClean = {
// fix individual rallies with rare errors
if (playerGender == 'women') {
dataPoints[126629].rally1 = '4#'; // '4n#'
dataPoints[145228].rally1 = '0#'; // '0xqqqs2f3s2#'
dataPoints[186203].rally1 = '6d'; // '6d5f28b2f3b3f;3s2m2d#' (separating 1st and 2nd serves)
dataPoints[186203].rally2 = '5f28b2f3b3f;3s2m2d#'; // null (separating 1st and 2nd serves)
}
if (playerGender == 'men') {
dataPoints[37539].rally1 = '4s2r1n@'; // '4s2dr1n@'
dataPoints[48918].rally1 = 'Q'; // 'q*'
dataPoints[50539].rally2 = '4sb+buw#'; // '4sb+bnuw#'
dataPoints[163419].rally1 = '6b27f+1ld#'; // '6db27f+1l#'
dataPoints[168639].rally2 = '6q2q1q2q2q2q2q3q2q1q3n@'; // '6q2q1q2q2q2q2q3q2q1g3n@'
dataPoints[172550].rally1 = '4n'; // '4n4b39s2b1f1b1f1b2b2b2f1b1f2b3b2b1w#' (separating 1st and 2nd serves)
dataPoints[172550].rally2 = '4b39s2b1f1b1f1b2b2b2f1b1f2b3b2b1w#'; // null (separating 1st and 2nd serves)
dataPoints[209116].rally1 = '6qn#'; // '6dn#'
dataPoints[210286].rally1 = '6w'; // '6W'
dataPoints[211520].rally1 = '6n'; // '6n4b81b2b2b3f2b1f1b3b2f1l2o2f2n#' (separating 1st and 2nd serves)
dataPoints[211520].rally2 = '4b81b2b2b3f2b1f1b3b2f1l2o2f2n#'; // null (separating 1st and 2nd serves)
dataPoints[212542].rally1 = '6qd#'; // 'c6!d#'
dataPoints[216575].rally1 = '4qn#'; // '4n#'
dataPoints[216607].rally1 = '4qn#'; // '4n#'
dataPoints[217965].rally1 = '6n'; // '6n5f!92d#' (separating 1st and 2nd serves)
dataPoints[217965].rally2 = '5f!92d#'; // null (separating 1st and 2nd serves)
dataPoints[226081].rally1 = '6f28f1f1b3-*'; // '6*f28f1f1b3-'
dataPoints[228177].rally1 = '6q3n#'; // '6n3n#'
dataPoints[234528].rally1 = 'V'; // '4V'
dataPoints[325639].rally1 = '6d'; // '6D'
dataPoints[325639].rally2 = '6s+38b3*'; // '6S+38b3*'
dataPoints[365098].rally1 = '5+n'; // '5+n0'
dataPoints[401651].rally2 = null; // '5+b18d@' (1st serve was 'R')
}
let endIdx, detIdx;
for (let p of dataPoints) {
// remove let serves and erroneous challenges
if (p.rally1.endsWith('c')) {
p.rally1 = p.rally1.slice(0, p.rally1.length-1) + 'C';
}
p.rally1 = p.rally1.replace(/c/g, '');
if (p.rally2) {
if (p.rally2.endsWith('c')) {
p.rally2 = p.rally2.slice(0, p.rally2.length-1) + 'C';
}
p.rally2 = p.rally2.replace(/c/g, '');
}
// move last shot details before the rally ending symbol
endIdx = p.rally1.search(/([\*nwdxge])(?!.*[\*nwdxge])/);
detIdx = p.rally1.search(/([\+\-\=\;\^\!0-9])(?!.*[\+\-\=\;\^\!0-9])/);
if ((endIdx >= 0) && (endIdx < detIdx)) {
p.rally1 = p.rally1.substring(0, endIdx) + p.rally1[detIdx] +
p.rally1.substring(endIdx + 1, detIdx) + p.rally1[endIdx] + p.rally1.substring(detIdx + 1);
}
if (p.rally2 && (p.rally2.length > 0)) {
endIdx = p.rally2.search(/([\*nwdxge])(?!.*[\*nwdxge])/);
detIdx = p.rally2.search(/([\+\-\=\;\^\!0-9])(?!.*[\+\-\=\;\^\!0-9])/);
if ((endIdx >= 0) && (endIdx < detIdx)) {
p.rally2 = p.rally2.substring(0, endIdx) + p.rally2[detIdx] +
p.rally2.substring(endIdx + 1, detIdx) + p.rally2[endIdx] + p.rally2.substring(detIdx + 1);
}
}
// replace erroneous rally endings in the middle of rallies with "q" (unknown shots)
if (p.rally1.length >= 5) {
p.rally1 = p.rally1.slice(0, -4).replace(/[nwdxge]/g, 'q') + p.rally1.slice(-4, p.rally1.length);
p.rally1 = p.rally1.slice(0, -2).replace(/\*/g, '+') + p.rally1.slice(-2, p.rally1.length);
}
if (p.rally2 && (p.rally2.length >= 5)) {
p.rally2 = p.rally2.slice(0, -4).replace(/[nwdxge]/g, 'q') + p.rally2.slice(-4, p.rally2.length);
p.rally2 = p.rally2.slice(0, -2).replace(/\*/g, '+') + p.rally2.slice(-2, p.rally2.length);
}
// replace erroneous serve with "0" (unknown direction)
if ((p.rally1.length > 1) && !['S', 'R', 'P', 'Q', '4', '5', '6', '0'].includes(p.rally1[0])) {
p.rally1 = '0' + p.rally1.slice(1);
}
if (p.rally2 && (p.rally2.length > 1) && !['S', 'R', 'P', 'Q', '4', '5', '6', '0'].includes(p.rally2[0])) {
p.rally2 = '0' + p.rally2.slice(1);
}
}
return dataPoints;
}
dataPoints = dataProcessed.pointData
dataMatches = dataProcessed.matchData
dataProcessed = {
let matchData = [];
let pointData = [];
let currentMatchName = '';
let currentMatchId = (playerGender == 'women') ? 99999 : -1; // for men start from 0, for women - from 100000
let matchItems;
for (let p of dataRaw) {
if (p.match_id != currentMatchName) {
currentMatchName = p.match_id;
currentMatchId += 1;
matchItems = currentMatchName.split('-');
matchData.push({
id: currentMatchId,
date: matchItems[0],
isMen: (matchItems[1] == 'M'),
tournamentName: (matchItems[2][matchItems[2].length-1] == '_') ?
matchItems[2].slice(0, matchItems[2].length - 1) : matchItems[2],
isRightyPlayer1: (matchItems[3][0] == 'R'),
isRightyPlayer2: (matchItems[3][1] == 'R'),
player1: (matchItems[4][matchItems[4].length-1] == '_') ?
matchItems[4].slice(0, matchItems[4].length - 1) : matchItems[4],
player2: (matchItems[5][matchItems[5].length-1] == '_') ?
matchItems[5].slice(0, matchItems[5].length - 1) : matchItems[5]
});
}
pointData.push({
matchId: currentMatchId,
tiebreakPointNumber: (p.TBpt) ? p.TBpt : 0,
isPlayer1Serving: (p.Svr == 1) ? 1 : 0,
rally1: p['1st'],
rally2: p['2nd'],
isPlayer1Winner: (p.PtWinner == 1) ? 1 : 0,
isGameWinner: (p.GmW > 0) ? 1 : 0,
isSetWinner: (p.SetW > 0) ? 1 : 0
});
}
return { matchData, pointData };
}
dataRaw = {
// read data from files
// men data
if (playerGender == 'men') {
const data_1 = d3.csvParse(await FileAttachment("tennis_pbp_men-1.csv").text(), d3.autoType);
const data_2 = d3.csvParse(await FileAttachment("tennis_pbp_men-2@2.csv").text(), d3.autoType);
const data_3 = d3.csvParse(await FileAttachment("tennis_pbp_men-3.csv").text(), d3.autoType);
const data_4 = d3.csvParse(await FileAttachment("tennis_pbp_men-4.csv").text(), d3.autoType);
const data_5 = d3.csvParse(await FileAttachment("tennis_pbp_men-5.csv").text(), d3.autoType);
const data_6 = d3.csvParse(await FileAttachment("tennis_pbp_men-6.csv").text(), d3.autoType);
const data_7 = d3.csvParse(await FileAttachment("tennis_pbp_men-7.csv").text(), d3.autoType);
const data_8 = d3.csvParse(await FileAttachment("tennis_pbp_men-8.csv").text(), d3.autoType);
const data_9 = d3.csvParse(await FileAttachment("tennis_pbp_men-9.csv").text(), d3.autoType);
return data_1.concat(data_2).concat(data_3).concat(data_4).concat(data_5)
.concat(data_6).concat(data_7).concat(data_8).concat(data_9);
}
// women data
if (playerGender == 'women') {
const data_1 = d3.csvParse(await FileAttachment("tennis_pbp_women-1.csv").text(), d3.autoType);
const data_2 = d3.csvParse(await FileAttachment("tennis_pbp_women-2.csv").text(), d3.autoType);
const data_3 = d3.csvParse(await FileAttachment("tennis_pbp_women-3.csv").text(), d3.autoType);
const data_4 = d3.csvParse(await FileAttachment("tennis_pbp_women-4.csv").text(), d3.autoType);
return data_1.concat(data_2).concat(data_3).concat(data_4);
}
}
playerGender = "men"
d3 = require("d3@5")
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.