534 lines
21 KiB
Python
534 lines
21 KiB
Python
#!/usr/bin/env python3
|
|
"""Parse Gramps JSONL and populate a well-structured Google Sheet."""
|
|
import json
|
|
import urllib.request
|
|
import urllib.parse
|
|
from collections import defaultdict
|
|
|
|
INPUT = "/root/.hermes/webui/attachments/8647800d8e2c/gramps-web-export-20260725204536.json"
|
|
SPREADSHEET_ID = "1uRokfwe3-9PyYlKY8Vnv3fkbQxIh_E2X_QIk3QpqN6c"
|
|
|
|
# ── Load data ──────────────────────────────────────────────────────────
|
|
objects = []
|
|
with open(INPUT) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
if "|" in line and line[0].isdigit():
|
|
line = line.split("|", 1)[1]
|
|
objects.append(json.loads(line))
|
|
|
|
by_handle = {o["handle"]: o for o in objects}
|
|
persons = [o for o in objects if o.get("_class") == "Person"]
|
|
families = [o for o in objects if o.get("_class") == "Family"]
|
|
events = [o for o in objects if o.get("_class") == "Event"]
|
|
places = [o for o in objects if o.get("_class") == "Place"]
|
|
media_list = [o for o in objects if o.get("_class") == "Media"]
|
|
|
|
EVENT_TYPES = {
|
|
0: "Mariage", 1: "Mariage", 2: "Divorce",
|
|
3: "Décès", 4: "Inhumation", 5: "Obsèques",
|
|
6: "Naissance", 7: "Baptême", 8: "Adoption",
|
|
9: "Baptême (LDS)", 10: "Endowment (LDS)",
|
|
11: "Scellement (LDS)", 12: "Naissance", 13: "Décès",
|
|
19: "Inhumation",
|
|
}
|
|
|
|
def event_type_str(t):
|
|
return EVENT_TYPES.get(t.get("value"), f"Type {t.get('value')}")
|
|
|
|
def format_date(d):
|
|
if not d:
|
|
return ""
|
|
dv = d.get("dateval", [0, 0, 0])
|
|
y, m, day = dv[2], dv[1], dv[0]
|
|
if y == 0:
|
|
return ""
|
|
if m == 0:
|
|
return str(y)
|
|
if day == 0:
|
|
months = ["", "Jan", "Fév", "Mar", "Avr", "Mai", "Juin",
|
|
"Juil", "Aoû", "Sep", "Oct", "Nov", "Déc"]
|
|
return f"{months[m]} {y}"
|
|
return f"{day:02d}/{m:02d}/{y}"
|
|
|
|
def person_name(p):
|
|
if not p:
|
|
return ""
|
|
n = p.get("primary_name", {})
|
|
first = n.get("first_name", "").strip()
|
|
sur_list = n.get("surname_list", [])
|
|
sur = sur_list[0].get("surname", "").strip() if sur_list else ""
|
|
suffix = n.get("suffix", "").strip()
|
|
parts = [first, sur, suffix] if suffix else [first, sur]
|
|
return " ".join(p for p in parts if p).strip()
|
|
|
|
def get_place_name(handle):
|
|
pl = by_handle.get(handle)
|
|
if pl:
|
|
return pl.get("name", {}).get("value", "")
|
|
return ""
|
|
|
|
def get_notes_for_person(p):
|
|
notes = []
|
|
for nh in p.get("note_list", []):
|
|
n = by_handle.get(nh)
|
|
if n:
|
|
notes.append(n.get("text", {}).get("string", ""))
|
|
return notes
|
|
|
|
def get_person_events(p):
|
|
birth, death = "", ""
|
|
for er in p.get("event_ref_list", []):
|
|
ev = by_handle.get(er.get("ref"))
|
|
if ev:
|
|
t = ev.get("type", {}).get("value")
|
|
d = format_date(ev.get("date"))
|
|
if t == 12:
|
|
birth = d
|
|
elif t == 13:
|
|
death = d
|
|
return birth, death
|
|
|
|
def get_parents(person):
|
|
father, mother = None, None
|
|
for pf in person.get("parent_family_list", []):
|
|
fam = by_handle.get(pf)
|
|
if fam:
|
|
if fam.get("father_handle"):
|
|
father = by_handle.get(fam["father_handle"])
|
|
if fam.get("mother_handle"):
|
|
mother = by_handle.get(fam["mother_handle"])
|
|
return father, mother
|
|
|
|
def get_children(family):
|
|
children = []
|
|
for cr in family.get("child_ref_list", []):
|
|
child = by_handle.get(cr.get("ref"))
|
|
if child:
|
|
children.append(child)
|
|
return children
|
|
|
|
def get_spouses(person):
|
|
spouses = []
|
|
for f in families:
|
|
if f.get("father_handle") == person["handle"]:
|
|
s = by_handle.get(f.get("mother_handle"))
|
|
if s:
|
|
spouses.append(s)
|
|
elif f.get("mother_handle") == person["handle"]:
|
|
s = by_handle.get(f.get("father_handle"))
|
|
if s:
|
|
spouses.append(s)
|
|
return spouses
|
|
|
|
def get_children_for_person(person):
|
|
all_children = []
|
|
for f in families:
|
|
if f.get("father_handle") == person["handle"] or f.get("mother_handle") == person["handle"]:
|
|
all_children.extend(get_children(f))
|
|
return all_children
|
|
|
|
# ── Determine branch ───────────────────────────────────────────────────
|
|
KELLER_ROOT = "a37f8646-d374-40da-863b-86febf23b165"
|
|
STAERLE_ROOT = "1c80dd1f-77b4-4ab9-940c-e0776b6161ff"
|
|
|
|
def compute_generation(person, root_handle, max_depth=20, visited=None):
|
|
if visited is None:
|
|
visited = set()
|
|
if person["handle"] in visited or max_depth <= 0:
|
|
return None
|
|
visited.add(person["handle"])
|
|
if person["handle"] == root_handle:
|
|
return 1
|
|
father, mother = get_parents(person)
|
|
for parent in [father, mother]:
|
|
if parent:
|
|
gen = compute_generation(parent, root_handle, max_depth - 1, visited)
|
|
if gen:
|
|
return gen + 1
|
|
return None
|
|
|
|
def get_branch(person, visited=None):
|
|
if visited is None:
|
|
visited = set()
|
|
if person["handle"] in visited:
|
|
return "Autre", 99
|
|
visited.add(person["handle"])
|
|
keller_gen = compute_generation(person, KELLER_ROOT)
|
|
staerle_gen = compute_generation(person, STAERLE_ROOT)
|
|
if keller_gen:
|
|
return "Keller", keller_gen
|
|
if staerle_gen:
|
|
return "Staerle", staerle_gen
|
|
for s in get_spouses(person):
|
|
sk, sg = get_branch(s, visited)
|
|
if sk and sk != "Autre":
|
|
return f"Allié ({sk})", sg + 1
|
|
return "Autre", 99
|
|
|
|
# ── Build data ──────────────────────────────────────────────────────────
|
|
|
|
# 1. PERSONNES
|
|
person_rows = [["ID", "Prénom", "Nom", "Nom complet", "Genre", "Naissance", "Décès",
|
|
"Branche", "Génération", "Père", "Mère",
|
|
"Conjoint(s)", "Enfants", "Profession/Notes"]]
|
|
person_branches = []
|
|
|
|
for p in persons:
|
|
pid = p.get("gramps_id", "")
|
|
n = p.get("primary_name", {})
|
|
first = n.get("first_name", "").strip()
|
|
sur_list = n.get("surname_list", [])
|
|
sur = sur_list[0].get("surname", "").strip() if sur_list else ""
|
|
gender_map = {0: "F", 1: "M", 2: "?"}
|
|
gender = gender_map.get(p.get("gender"), "?")
|
|
birth, death = get_person_events(p)
|
|
branch, gen = get_branch(p)
|
|
father, mother = get_parents(p)
|
|
father_name = person_name(father) if father else ""
|
|
mother_name = person_name(mother) if mother else ""
|
|
spouses = get_spouses(p)
|
|
spouse_names = ", ".join(person_name(s) for s in spouses)
|
|
children = get_children_for_person(p)
|
|
child_names = ", ".join(person_name(c) for c in children)
|
|
notes = "; ".join(get_notes_for_person(p))
|
|
person_rows.append([pid, first, sur, f"{first} {sur}", gender, birth, death,
|
|
branch, gen, father_name, mother_name,
|
|
spouse_names, child_names, notes])
|
|
person_branches.append(branch)
|
|
|
|
# 2. FAMILLES
|
|
family_rows = [["ID", "Père", "Mère", "Type", "Date mariage", "Lieu mariage", "Enfants"]]
|
|
for f in families:
|
|
fid = f.get("gramps_id", "")
|
|
father = by_handle.get(f.get("father_handle"))
|
|
mother = by_handle.get(f.get("mother_handle"))
|
|
f_name = person_name(father) if father else "?"
|
|
m_name = person_name(mother) if mother else "?"
|
|
rel_type = f.get("type", {}).get("value", 0)
|
|
rel_str = "Marié" if rel_type == 0 else "Non marié" if rel_type == 3 else ""
|
|
marriage_date, marriage_place = "", ""
|
|
for er in f.get("event_ref_list", []):
|
|
ev = by_handle.get(er.get("ref"))
|
|
if ev:
|
|
marriage_date = format_date(ev.get("date"))
|
|
marriage_place = get_place_name(ev.get("place", ""))
|
|
children = get_children(f)
|
|
enfants = ", ".join(person_name(c) for c in children)
|
|
family_rows.append([fid, f_name, m_name, rel_str, marriage_date, marriage_place, enfants])
|
|
|
|
# 3. ÉVÉNEMENTS
|
|
event_rows = [["ID", "Type", "Date", "Personne", "Lieu", "Description"]]
|
|
for ev in events:
|
|
eid = ev.get("gramps_id", "")
|
|
etype = event_type_str(ev.get("type", {}))
|
|
date = format_date(ev.get("date"))
|
|
place = get_place_name(ev.get("place", ""))
|
|
desc = ev.get("description", "")
|
|
person = ""
|
|
for p in persons:
|
|
for er in p.get("event_ref_list", []):
|
|
if er.get("ref") == ev["handle"]:
|
|
person = person_name(p)
|
|
break
|
|
if person:
|
|
break
|
|
event_rows.append([eid, etype, date, person, place, desc])
|
|
|
|
# 4. LIEUX
|
|
place_rows = [["ID", "Nom", "Type", "Latitude", "Longitude", "Fait partie de"]]
|
|
for pl in places:
|
|
pid = pl.get("gramps_id", "")
|
|
name = pl.get("name", {}).get("value", "")
|
|
ptype = pl.get("place_type", {}).get("string", "")
|
|
lat = pl.get("lat", "")
|
|
lon = pl.get("long", "")
|
|
parent_place = ""
|
|
for pr in pl.get("placeref_list", []):
|
|
parent = by_handle.get(pr.get("ref"))
|
|
if parent:
|
|
parent_place = parent.get("name", {}).get("value", "")
|
|
place_rows.append([pid, name, ptype, lat, lon, parent_place])
|
|
|
|
# 5. MÉDIAS
|
|
media_rows = [["ID", "Description", "Type MIME", "Chemin", "Date"]]
|
|
for m in media_list:
|
|
mid = m.get("gramps_id", "")
|
|
desc = m.get("desc", "")
|
|
mime = m.get("mime", "")
|
|
path = m.get("path", "")
|
|
date_str = format_date(m.get("date", {}))
|
|
media_rows.append([mid, desc, mime, path, date_str])
|
|
|
|
# 6. STATISTIQUES
|
|
stats_rows = [
|
|
["Métrique", "Valeur"],
|
|
["Total personnes", len(persons)],
|
|
["Hommes", sum(1 for p in persons if p.get("gender") == 1)],
|
|
["Femmes", sum(1 for p in persons if p.get("gender") == 0)],
|
|
["Total familles", len(families)],
|
|
["Familles mariées", sum(1 for f in families if f.get("type", {}).get("value") == 0)],
|
|
["Total événements", len(events)],
|
|
["Naissances", sum(1 for ev in events if ev.get("type", {}).get("value") in (6, 12))],
|
|
["Décès", sum(1 for ev in events if ev.get("type", {}).get("value") in (3, 13))],
|
|
["Mariages", sum(1 for ev in events if ev.get("type", {}).get("value") in (0, 1))],
|
|
["Total lieux", len(places)],
|
|
["Total médias", len(media_list)],
|
|
["", ""],
|
|
["Branche Keller", ""],
|
|
[" Personnes Keller", sum(1 for b in person_branches if b == "Keller")],
|
|
["Branche Staerle", ""],
|
|
[" Personnes Staerle", sum(1 for b in person_branches if b and "Staerle" in b)],
|
|
]
|
|
|
|
# ── Write to Sheets ────────────────────────────────────────────────────
|
|
TOKEN = open("/tmp/gsheet_token.txt").read().strip()
|
|
|
|
def get_sheets():
|
|
url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}?fields=sheets.properties"
|
|
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {TOKEN}"})
|
|
resp = json.loads(urllib.request.urlopen(req).read())
|
|
return resp.get("sheets", [])
|
|
|
|
def get_sheet_id(sheets, name):
|
|
for s in sheets:
|
|
if s["properties"]["title"] == name:
|
|
return s["properties"]["sheetId"]
|
|
return None
|
|
|
|
def clear_sheet(sheet_id):
|
|
"""Clear all content from a sheet."""
|
|
body = json.dumps({
|
|
"requests": [{
|
|
"updateCells": {
|
|
"range": {"sheetId": sheet_id},
|
|
"fields": "userEnteredValue"
|
|
}
|
|
}]
|
|
})
|
|
url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}:batchUpdate"
|
|
req = urllib.request.Request(url, data=body.encode(),
|
|
method="POST",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
urllib.request.urlopen(req)
|
|
|
|
def add_sheet(title):
|
|
"""Add a new sheet, return its ID."""
|
|
body = json.dumps({
|
|
"requests": [{
|
|
"addSheet": {
|
|
"properties": {
|
|
"title": title,
|
|
"gridProperties": {"rowCount": 2000, "columnCount": 26}
|
|
}
|
|
}
|
|
}]
|
|
})
|
|
url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}:batchUpdate"
|
|
req = urllib.request.Request(url, data=body.encode(),
|
|
method="POST",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
resp = json.loads(urllib.request.urlopen(req).read())
|
|
return resp["replies"][0]["addSheet"]["properties"]["sheetId"]
|
|
|
|
def write_sheet(sheet_id, rows):
|
|
"""Write rows to a sheet by ID."""
|
|
# Clear first
|
|
clear_sheet(sheet_id)
|
|
|
|
# Write data
|
|
body = json.dumps({"values": rows, "majorDimension": "ROWS"})
|
|
# Use the sheet title from the ID — we need to find it
|
|
write_url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}/values/A1?valueInputOption=USER_ENTERED"
|
|
write_req = urllib.request.Request(write_url, data=body.encode(),
|
|
method="PUT",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
resp = json.loads(urllib.request.urlopen(write_req).read())
|
|
return resp.get('updatedCells', 0)
|
|
|
|
def write_sheet_by_name(sheet_name, rows):
|
|
"""Write rows to a sheet by name."""
|
|
encoded_name = urllib.parse.quote(sheet_name)
|
|
range_name = f"{encoded_name}!A1"
|
|
body = json.dumps({"values": rows, "majorDimension": "ROWS"})
|
|
write_url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}/values/{range_name}?valueInputOption=USER_ENTERED"
|
|
write_req = urllib.request.Request(write_url, data=body.encode(),
|
|
method="PUT",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
resp = json.loads(urllib.request.urlopen(write_req).read())
|
|
return resp.get('updatedCells', 0)
|
|
|
|
def format_sheet(sheet_id, num_cols, num_rows, name="Sheet", alt_color=None):
|
|
"""Apply formatting: frozen header, auto-filter, alternating colors."""
|
|
requests = []
|
|
|
|
# Freeze header row
|
|
requests.append({
|
|
"updateSheetProperties": {
|
|
"properties": {
|
|
"sheetId": sheet_id,
|
|
"gridProperties": {"frozenRowCount": 1}
|
|
},
|
|
"fields": "gridProperties.frozenRowCount"
|
|
}
|
|
})
|
|
|
|
# Auto-filter
|
|
if num_rows > 1:
|
|
requests.append({
|
|
"setBasicFilter": {
|
|
"filter": {
|
|
"range": {
|
|
"sheetId": sheet_id,
|
|
"startRowIndex": 0,
|
|
"endRowIndex": num_rows,
|
|
"startColumnIndex": 0,
|
|
"endColumnIndex": num_cols
|
|
}
|
|
}
|
|
}
|
|
})
|
|
|
|
# Header: bold white on dark blue
|
|
requests.append({
|
|
"repeatCell": {
|
|
"range": {
|
|
"sheetId": sheet_id,
|
|
"startRowIndex": 0, "endRowIndex": 1,
|
|
"startColumnIndex": 0, "endColumnIndex": num_cols
|
|
},
|
|
"cell": {
|
|
"userEnteredFormat": {
|
|
"textFormat": {"bold": True, "foregroundColor": {"red": 1, "green": 1, "blue": 1}},
|
|
"backgroundColor": {"red": 0.15, "green": 0.23, "blue": 0.35},
|
|
"verticalAlignment": "MIDDLE"
|
|
}
|
|
},
|
|
"fields": "userEnteredFormat(textFormat,backgroundColor,verticalAlignment)"
|
|
}
|
|
})
|
|
|
|
# Alternating row colors
|
|
if alt_color and num_rows > 1:
|
|
for i in range(1, num_rows):
|
|
if i % 2 == 0:
|
|
requests.append({
|
|
"repeatCell": {
|
|
"range": {
|
|
"sheetId": sheet_id,
|
|
"startRowIndex": i, "endRowIndex": i + 1,
|
|
"startColumnIndex": 0, "endColumnIndex": num_cols
|
|
},
|
|
"cell": {"userEnteredFormat": {"backgroundColor": alt_color}},
|
|
"fields": "userEnteredFormat.backgroundColor"
|
|
}
|
|
})
|
|
|
|
# Auto-resize columns
|
|
requests.append({
|
|
"autoResizeDimensions": {
|
|
"dimensions": {
|
|
"sheetId": sheet_id,
|
|
"dimension": "COLUMNS",
|
|
"startIndex": 0, "endIndex": num_cols
|
|
}
|
|
}
|
|
})
|
|
|
|
body = json.dumps({"requests": requests})
|
|
url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}:batchUpdate"
|
|
req = urllib.request.Request(url, data=body.encode(),
|
|
method="POST",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
urllib.request.urlopen(req)
|
|
print(f" ✓ {name}: formatted")
|
|
|
|
# ── Main ───────────────────────────────────────────────────────────────
|
|
print("Setting up sheets...")
|
|
sheets = get_sheets()
|
|
existing_names = {s["properties"]["title"] for s in sheets}
|
|
|
|
# Add missing sheets
|
|
for name in ["Médias", "Statistiques"]:
|
|
if name not in existing_names:
|
|
add_sheet(name)
|
|
print(f" ✓ Added sheet '{name}'")
|
|
|
|
# Write data to all sheets
|
|
print("\nWriting data...")
|
|
sheets = get_sheets()
|
|
|
|
for name, rows in [("Personnes", person_rows), ("Familles", family_rows),
|
|
("Événements", event_rows), ("Lieux", place_rows),
|
|
("Médias", media_rows), ("Statistiques", stats_rows)]:
|
|
sid = get_sheet_id(sheets, name)
|
|
if sid:
|
|
cells = write_sheet_by_name(name, rows)
|
|
print(f" ✓ {name}: {cells} cells written")
|
|
else:
|
|
print(f" ✗ {name}: sheet not found")
|
|
|
|
# Format all sheets
|
|
print("\nFormatting...")
|
|
sheets = get_sheets()
|
|
|
|
formats = [
|
|
("Personnes", person_rows, {"red": 0.95, "green": 0.97, "blue": 1.0}),
|
|
("Familles", family_rows, {"red": 1.0, "green": 0.97, "blue": 0.95}),
|
|
("Événements", event_rows, {"red": 0.95, "green": 0.95, "blue": 0.95}),
|
|
("Lieux", place_rows, {"red": 0.97, "green": 0.97, "blue": 0.95}),
|
|
("Médias", media_rows, {"red": 0.95, "green": 0.95, "blue": 0.97}),
|
|
("Statistiques", stats_rows, {"red": 0.95, "green": 0.95, "blue": 0.95}),
|
|
]
|
|
for name, rows, color in formats:
|
|
sid = get_sheet_id(sheets, name)
|
|
if sid:
|
|
format_sheet(sid, len(rows[0]), len(rows), name=name, alt_color=color)
|
|
|
|
# ── Color-code by branch on Personnes ──────────────────────────────────
|
|
print("\nColoring by branch...")
|
|
sid_p = get_sheet_id(sheets, "Personnes")
|
|
if sid_p:
|
|
branch_colors = {
|
|
"Keller": {"red": 0.85, "green": 0.92, "blue": 1.0},
|
|
"Staerle": {"red": 0.85, "green": 1.0, "blue": 0.85},
|
|
}
|
|
for i, branch in enumerate(person_branches):
|
|
row_idx = i + 1
|
|
color = None
|
|
for key, c in branch_colors.items():
|
|
if branch and key in branch:
|
|
color = c
|
|
break
|
|
if color:
|
|
req_body = json.dumps({
|
|
"requests": [{
|
|
"repeatCell": {
|
|
"range": {
|
|
"sheetId": sid_p,
|
|
"startRowIndex": row_idx, "endRowIndex": row_idx + 1,
|
|
"startColumnIndex": 0, "endColumnIndex": len(person_rows[0])
|
|
},
|
|
"cell": {"userEnteredFormat": {"backgroundColor": color}},
|
|
"fields": "userEnteredFormat.backgroundColor"
|
|
}
|
|
}]
|
|
})
|
|
url = f"https://sheets.googleapis.com/v4/spreadsheets/{SPREADSHEET_ID}:batchUpdate"
|
|
req = urllib.request.Request(url, data=req_body.encode(),
|
|
method="POST",
|
|
headers={"Authorization": f"Bearer {TOKEN}",
|
|
"Content-Type": "application/json"})
|
|
urllib.request.urlopen(req)
|
|
print(f" ✓ Personnes: {len(person_branches)} rows colored by branch")
|
|
|
|
print(f"\n✅ Done! Open: https://docs.google.com/spreadsheets/d/{SPREADSHEET_ID}/edit")
|
|
print(f" Personnes: {len(person_rows)-1} | Familles: {len(family_rows)-1} | Événements: {len(event_rows)-1}")
|
|
print(f" Lieux: {len(place_rows)-1} | Médias: {len(media_rows)-1} | Statistiques: {len(stats_rows)-1}")
|