#Parser to convert the Datini onomastics CSV file into TTL format # Utilities to read/write csv files import csv # Utilities to handle character encodings import unicodedata # Ordered Dicts from collections import OrderedDict import json import re # OPZIONAL IMPORTS # For timestamping/simple speed tests from datetime import datetime # Random number generator from random import * # System & command line utilities import sys # Json for the dictionary import json import_dir = '/Users/federicaspinelli/TEAMOVI/Parser/DATA/ASPO/CSV/ospedale/' export_dir = '/Users/federicaspinelli/TEAMOVI/Parser/DATA/ASPO/RDF/ospedale/' # Custom class to store URIs + related infos for the ontologies/repositories class RDFcoords: def __init__(self, uri, prefix, code = None): self.uri = uri self.prefix = prefix self.code = code # Repositories aspoCoords = RDFcoords('', 'aspo:') foafCoords = RDFcoords('', 'foaf:') cidocCoords = RDFcoords('', 'crm:') schemaCoords = RDFcoords('', 'schema:') personCoords = RDFcoords('', 'person:') nsCoords = RDFcoords('', 'rdf:') rdfsCoords = RDFcoords('', 'rdfs:') owlCoords = RDFcoords('', 'owl:') # Basic functions for triples / shortened triples in TTL format def triple(subject, predicate, object1): line = subject + ' ' + predicate + ' ' + object1 return line def doublet(predicate, object1): line = ' ' + predicate + ' ' + object1 return line def singlet(object1): line = ' ' + object1 return line # Line endings in TTL format continueLine1 = ' ;\n' continueLine2 = ' ,\n' closeLine = ' .\n' def writeTTLHeader(output): output.write('@prefix ' + aspoCoords.prefix + ' ' + aspoCoords.uri + closeLine) output.write('@prefix ' + foafCoords.prefix + ' ' + foafCoords.uri + closeLine) output.write('@prefix ' + cidocCoords.prefix + ' ' + cidocCoords.uri + closeLine) output.write('@prefix ' + personCoords.prefix + ' ' + personCoords.uri + closeLine) output.write('@prefix ' + schemaCoords.prefix + ' ' + schemaCoords.uri + closeLine) output.write('@prefix ' + nsCoords.prefix + ' ' + nsCoords.uri + closeLine) output.write('@prefix ' + rdfsCoords.prefix + ' ' + rdfsCoords.uri + closeLine) output.write('@prefix ' + owlCoords.prefix + ' ' + owlCoords.uri + closeLine) output.write('\n') filePrefix = 'OSPEDALE-onomastica' fileType = '-persone-singole-occupation' max_entries = 10000000000000 with open(import_dir + filePrefix + fileType + '.csv', newline="") as csv_file, open( export_dir + filePrefix + fileType + '.ttl', 'w') as output: reader = csv.DictReader(csv_file) writeTTLHeader(output) first = True ii = 0 for row in reader: # The index ii is used to process a limited number of entries for testing purposes ii = ii + 1 id_aspo = row['recordid'] #placeHolders aspoPlaceHolder = aspoCoords.prefix + id_aspo if row['occupation_1'] != '' and row['occupation_1'] != ' ' : occupazioni = [] pipe = "|" if pipe in row['occupation_1']: occupazioni = row['occupation_1'].split('|') for occupazione in occupazioni: #Remove all white-space characters: txt = occupazione x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_1'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['ID_ente_1'] == '' and row['place occupation 1 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 1 ENTE']: placeoccupazioni = row['place occupation 1 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 1 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_1'].lower() + ' presso ' + row['place occupation 1 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 1 ENTE'].split('|') txt = row['place occupation 1 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_1'].lower() + ' presso ' + row['place occupation 1 ENTE'].lower() + '\"') + closeLine output.write(line) else: #Remove all white-space characters: txt = row['occupation_1'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_1'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['ID_ente_1'] == '' and row['place occupation 1 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 1 ENTE']: placeoccupazioni = row['place occupation 1 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 1 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_1'].lower() + ' presso ' + row['place occupation 1 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 1 ENTE'].split('|') txt = row['place occupation 1 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_1'].lower() + ' presso ' + row['place occupation 1 ENTE'].lower() + '\"') + closeLine output.write(line) if row['occupation_2'] != '' and row['occupation_2'] != ' ' : occupazioni = [] pipe = "|" if pipe in row['occupation_2']: occupazioni = row['occupation_2'].split('|') for occupazione in occupazioni: #Remove all white-space characters: txt = occupazione x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_2'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['place occupation 2 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 2 ENTE']: placeoccupazioni = row['place occupation 2 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 2 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_2'].lower() + ' presso ' + row['place occupation 2 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 2 ENTE'].split('|') txt = row['place occupation 2 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_2'].lower() + ' presso ' + row['place occupation 2 ENTE'].lower() + '\"') + closeLine output.write(line) else: #Remove all white-space characters: txt = row['occupation_2'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_2'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['ID_ente_2'] == '' and row['place occupation 2 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 2 ENTE']: placeoccupazioni = row['place occupation 2 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 2 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_2'].lower() + ' presso ' + row['place occupation 2 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 2 ENTE'].split('|') txt = row['place occupation 2 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_2'].lower() + ' presso ' + row['place occupation 2 ENTE'].lower() + '\"') + closeLine output.write(line) if row['occupation_3'] != '' and row['occupation_3'] != ' ' : occupazioni = [] pipe = "|" if pipe in row['occupation_3']: occupazioni = row['occupation_3'].split('|') for occupazione in occupazioni: #Remove all white-space characters: txt = occupazione x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_3'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['ID_ente_3'] == '' and row['place occupation 3 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 3 ENTE']: placeoccupazioni = row['place occupation 3 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 3 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_3'].lower() + ' presso ' + row['place occupation 3 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 3 ENTE'].split('|') txt = row['place occupation 3 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_3'].lower() + ' presso ' + row['place occupation 3 ENTE'].lower() + '\"') + closeLine output.write(line) else: #Remove all white-space characters: txt = row['occupation_3'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) occ = re.sub(r'[^A-Za-z]','', y) occupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'hasOccupation', occupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, nsCoords.prefix + 'type', schemaCoords.prefix + 'Occupation') + closeLine output.write(line) line = triple(occupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) if row['ID_ente_3'] != '': placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) elif row['ID_ente_3'] == '' and row['place occupation 3 ENTE'] != '': placeoccupazioni = [] pipe = "|" if pipe in row['place occupation 3 ENTE']: placeoccupazioni = row['place occupation 3 ENTE'].split('|') for placeoccupazione in placeoccupazioni: #Remove all white-space characters: txt = row['place occupation 3 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_3'].lower() + ' presso ' + row['place occupation 3 ENTE'].lower() + '\"') + closeLine output.write(line) else: placeoccupazioni = row['place occupation 3 ENTE'].split('|') txt = row['place occupation 3 ENTE'] x = re.sub("\n", " ", txt) y = re.sub("\s\s", "", x) placeoccupationPlaceHolder = '' line = triple(aspoPlaceHolder, schemaCoords.prefix + 'workLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(occupationPlaceHolder, schemaCoords.prefix + 'occupationLocation', placeoccupationPlaceHolder) + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, rdfsCoords.prefix + 'label', '\"' + y + '\"') + closeLine output.write(line) line = triple(placeoccupationPlaceHolder, nsCoords.prefix + 'type', cidocCoords.prefix + 'E53_Place') + closeLine output.write(line) line = triple(aspoPlaceHolder, schemaCoords.prefix + 'jobTitle', '\"' + row['occupation_3'].lower() + ' presso ' + row['place occupation 3 ENTE'].lower() + '\"') + closeLine output.write(line) output.write('\n') # # # Limit number of entries processed (if desired) if (ii > max_entries): break