123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357 |
- #Parser to convert the Datini onomastics CSV file into TTL format
- # Utilities to read/write csv files
- import csv
- # Utilities to handle character encodings
- import unicodedata
- # Ordered Dicts
- from collections import OrderedDict
- import json
- import re
- # OPZIONAL IMPORTS
- # For timestamping/simple speed tests
- from datetime import datetime
- # Random number generator
- from random import *
- # System & command line utilities
- import sys
- # Json for the dictionary
- import json
- import_dir = '/Users/federicaspinelli/TEAMOVI/Parser/DATA/ASPO/CSV/ospedale/'
- export_dir = '/Users/federicaspinelli/TEAMOVI/Parser/DATA/ASPO/RDF/ospedale/'
- # Custom class to store URIs + related infos for the ontologies/repositories
- class RDFcoords:
- def __init__(self, uri, prefix, code = None):
- self.uri = uri
- self.prefix = prefix
- self.code = code
- # Repositories
- aspoCoords = RDFcoords('<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/>', 'aspo:')
- foafCoords = RDFcoords('<http://xmlns.com/foaf/0.1/>', 'foaf:')
- cidocCoords = RDFcoords('<http://www.cidoc-crm.org/cidoc-crm/>', 'crm:')
- schemaCoords = RDFcoords('<http://schema.org/>', 'schema:')
- personCoords = RDFcoords('<http://www.w3.org/ns/person#>', 'person:')
- nsCoords = RDFcoords('<http://www.w3.org/1999/02/22-rdf-syntax-ns#>', 'rdf:')
- rdfsCoords = RDFcoords('<http://www.w3.org/2000/01/rdf-schema#>', 'rdfs:')
- owlCoords = RDFcoords('<http://www.w3.org/2002/07/owl#>', 'owl:')
- # Basic functions for triples / shortened triples in TTL format
- def triple(subject, predicate, object1):
- line = subject + ' ' + predicate + ' ' + object1
- return line
- def doublet(predicate, object1):
- line = ' ' + predicate + ' ' + object1
- return line
- def singlet(object1):
- line = ' ' + object1
- return line
- # Line endings in TTL format
- continueLine1 = ' ;\n'
- continueLine2 = ' ,\n'
- closeLine = ' .\n'
- def writeTTLHeader(output):
- output.write('@prefix ' + aspoCoords.prefix + ' ' + aspoCoords.uri + closeLine)
- output.write('@prefix ' + foafCoords.prefix + ' ' + foafCoords.uri + closeLine)
- output.write('@prefix ' + cidocCoords.prefix + ' ' + cidocCoords.uri + closeLine)
- output.write('@prefix ' + personCoords.prefix + ' ' + personCoords.uri + closeLine)
- output.write('@prefix ' + schemaCoords.prefix + ' ' + schemaCoords.uri + closeLine)
- output.write('@prefix ' + nsCoords.prefix + ' ' + nsCoords.uri + closeLine)
- output.write('@prefix ' + rdfsCoords.prefix + ' ' + rdfsCoords.uri + closeLine)
- output.write('@prefix ' + owlCoords.prefix + ' ' + owlCoords.uri + closeLine)
- output.write('\n')
- filePrefix = 'OSPEDALE - onomastica '
- fileType = '- corporate_body'
- max_entries = 10000000000000
- with open(import_dir + filePrefix + fileType + '.csv', newline="") as csv_file, open(
- export_dir + filePrefix + fileType + '.ttl', 'w') as output:
- reader = csv.DictReader(csv_file)
- writeTTLHeader(output)
- first = True
- ii = 0
- for row in reader:
- # The index ii is used to process a limited number of entries for testing purposes
- ii = ii + 1
- if row['entityType'] == 'corporateBody':
- id_aspo = row['recordId']
- aspoPlaceHolder = aspoCoords.prefix + id_aspo
- line = triple(aspoPlaceHolder,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E74_Group') + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder,
- nsCoords.prefix + 'type',
- foafCoords.prefix + 'Group') + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder,
- cidocCoords.prefix + 'P1_is_identified_by',
- aspoPlaceHolder + "_E42") + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder + "_E42",
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E42_Identifier') + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder + "_E42",
- rdfsCoords.prefix + 'label',
- '\"' + id_aspo + '\"') + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder,
- foafCoords.prefix + 'name',
- '\"' + row['nameEntry@normal'].replace("\"", "") + '\"') + closeLine
- output.write(line)
- line = triple(aspoPlaceHolder,
- rdfsCoords.prefix + 'label',
- '\"' + row['nameEntry@normal'].replace("\"", "") + '\"') + closeLine
- output.write(line)
- if row['occupation'] != '' and row['occupation'] != ' ' :
- occupazioni = []
- pipe = "|"
- if pipe in row['occupation']:
- occupazioni = row['occupation'].split('|')
- for occupazione in occupazioni:
- #Remove all white-space characters:
- txt = occupazione
- x = re.sub("\n", " ", txt)
- y = re.sub("\s\s", "", x)
- occ = re.sub(r'[^A-Za-z]','', y)
- occupationPlaceHolder = '<http://www.archiviodistato.prato.it/' + occ.replace(" ","_") + '>'
- line = triple(aspoPlaceHolder,
- schemaCoords.prefix + 'hasOccupation',
- occupationPlaceHolder) + closeLine
- output.write(line)
- line = triple(occupationPlaceHolder,
- nsCoords.prefix + 'type',
- schemaCoords.prefix + 'Occupation') + closeLine
- output.write(line)
- line = triple(occupationPlaceHolder,
- rdfsCoords.prefix + 'label',
- '\"' + y + '\"') + closeLine
- output.write(line)
- else:
- #Remove all white-space characters:
- txt = row['occupation']
- x = re.sub("\n", " ", txt)
- y = re.sub("\s\s", "", x)
- occ = re.sub(r'[^A-Za-z]','', y)
- occupationPlaceHolder = '<http://www.archiviodistato.prato.it/' + occ.replace(" ","_") + '>'
- line = triple(aspoPlaceHolder,
- schemaCoords.prefix + 'hasOccupation',
- occupationPlaceHolder) + closeLine
- output.write(line)
- line = triple(occupationPlaceHolder,
- nsCoords.prefix + 'type',
- schemaCoords.prefix + 'Occupation') + closeLine
- output.write(line)
- line = triple(occupationPlaceHolder,
- rdfsCoords.prefix + 'label',
- '\"' + y + '\"') + closeLine
- output.write(line)
-
- if row['tipologia'] != '':
- line = triple(aspoPlaceHolder,
- cidocCoords.prefix + 'P2_has_type',
- '\"' + row['tipologia'] + '\"') + closeLine
- output.write(line)
-
- if row['intestatario principale: nome o ID se lo abbiamo'] != '':
- E13placeHolder = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['recordId'] + '_S_' + row['intestatario principale: nome o ID se lo abbiamo'].replace(' ', '_') + ">"
- line = triple(E13placeHolder,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E13_Attribute_Assignment') + closeLine
- output.write(line)
- line = triple(E13placeHolder, cidocCoords.prefix + 'P141_assigned', aspoPlaceHolder) + closeLine
- output.write(line)
- line = triple(E13placeHolder,
- rdfsCoords.prefix + 'label',
- '\"Relazione: ' + row['intestatario principale: nome o ID se lo abbiamo'] + ' intestatario principale del gruppo ' + row['recordId'] + '\"') + closeLine
- output.write(line)
- E55placeHolder = '<http://www.archiviodistato.prato.it/socio_principale>'
- line = triple(E13placeHolder, cidocCoords.prefix + 'P42_assigned', E55placeHolder) + closeLine
- output.write(line)
- line = triple(E55placeHolder, rdfsCoords.prefix + 'label', '\"Socio\"') + closeLine
- output.write(line)
- line = triple(E55placeHolder, cidocCoords.prefix + 'P3_has_note', '\"Intestatario principale\"') + closeLine
- output.write(line)
- id = row['intestatario principale: nome o ID se lo abbiamo']
- if re.match(r'IT-ASPO', id):
- socioid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['intestatario principale: nome o ID se lo abbiamo']+ ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socioid) + closeLine
- output.write(line)
- line = triple(socioid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- else:
- socionoid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['intestatario principale: nome o ID se lo abbiamo'].replace(' ', '_').lower() + ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socionoid) + closeLine
- output.write(line)
- line = triple(socionoid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- line = triple(socionoid,
- rdfsCoords.prefix + 'label',
- '\"' + row['intestatario principale: nome o ID se lo abbiamo'] + '\"') + closeLine
- output.write(line)
-
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E21_Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- personCoords.prefix + 'Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- foafCoords.prefix + 'person') + closeLine
- output.write(line)
- if row['socio 2'] != '':
- E13placeHolder = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['recordId'] + '_S_' + row['socio 2'].replace(' ', '_') + ">"
- line = triple(E13placeHolder,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E13_Attribute_Assignment') + closeLine
- output.write(line)
- line = triple(E13placeHolder, cidocCoords.prefix + 'P141_assigned', aspoPlaceHolder) + closeLine
- output.write(line)
- line = triple(E13placeHolder,
- rdfsCoords.prefix + 'label',
- '\"Relazione: ' + row['socio 2'] + ' socio del gruppo ' + row['recordId'] + '\"') + closeLine
- output.write(line)
- E55placeHolder = '<http://www.archiviodistato.prato.it/socio>'
- line = triple(E13placeHolder, cidocCoords.prefix + 'P42_assigned', E55placeHolder) + closeLine
- output.write(line)
- line = triple(E55placeHolder, rdfsCoords.prefix + 'label', '\"Socio\"') + closeLine
- output.write(line)
- id = row['socio 2']
- if re.match(r'IT-ASPO', id):
- socioid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['socio 2']+ ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socioid) + closeLine
- output.write(line)
- line = triple(socioid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- else:
- socionoid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['socio 2'].replace(' ', '_').lower() + ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socionoid) + closeLine
- output.write(line)
- line = triple(socionoid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- line = triple(socionoid,
- rdfsCoords.prefix + 'label',
- '\"' + row['socio 2'] + '\"') + closeLine
- output.write(line)
-
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E21_Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- personCoords.prefix + 'Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- foafCoords.prefix + 'person') + closeLine
- output.write(line)
-
-
- E13placeHolder = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['recordId'] + '_S_' + row['socio 4'].replace(' ', '_') + ">"
- line = triple(E13placeHolder,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E13_Attribute_Assignment') + closeLine
- output.write(line)
- line = triple(E13placeHolder, cidocCoords.prefix + 'P141_assigned', aspoPlaceHolder) + closeLine
- output.write(line)
- line = triple(E13placeHolder,
- rdfsCoords.prefix + 'label',
- '\"Relazione: ' + row['socio 4'] + ' socio del gruppo ' + row['recordId'] + '\"') + closeLine
- output.write(line)
- E55placeHolder = '<http://www.archiviodistato.prato.it/socio>'
- line = triple(E13placeHolder, cidocCoords.prefix + 'P42_assigned', E55placeHolder) + closeLine
- output.write(line)
- line = triple(E55placeHolder, rdfsCoords.prefix + 'label', '\"Socio\"') + closeLine
- output.write(line)
- id = row['socio 4']
- if re.match(r'IT-ASPO', id):
- socioid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['socio 4']+ ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socioid) + closeLine
- output.write(line)
- line = triple(socioid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- else:
- socionoid = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['socio 4'].replace(' ', '_').lower() + ">"
- line = triple(aspoPlaceHolder, cidocCoords.prefix + 'P107_has_current_or_former_member', socionoid) + closeLine
- output.write(line)
- line = triple(socionoid, cidocCoords.prefix + 'P141_assigned', E13placeHolder) + closeLine
- output.write(line)
- line = triple(socionoid,
- rdfsCoords.prefix + 'label',
- '\"' + row['socio 4'] + '\"') + closeLine
- output.write(line)
-
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E21_Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- personCoords.prefix + 'Person') + closeLine
- output.write(line)
- line = triple(socionoid,
- nsCoords.prefix + 'type',
- foafCoords.prefix + 'person') + closeLine
- output.write(line)
-
- if row['sede operativa'] != '':
- if row['id sede operativa'] != '':
- E13placeHolder = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + row['id sede operativa'] + '/SO' + ">"
- line = triple(E13placeHolder,
- nsCoords.prefix + 'type',
- cidocCoords.prefix + 'E13_Attribute_Assignment') + closeLine
- output.write(line)
- line = triple(E13placeHolder, cidocCoords.prefix + 'P141_assigned', aspoPlaceHolder) + closeLine
- output.write(line)
- line = triple(E13placeHolder,
- rdfsCoords.prefix + 'label',
- '\" ' + row['sede operativa'] + ' sede operativa del gruppo ' + row['recordId'] + '\"') + closeLine
- output.write(line)
- E55placeHolder = '<http://www.archiviodistato.prato.it/sede_operativa>'
- line = triple(E13placeHolder, cidocCoords.prefix + 'P42_assigned', E55placeHolder) + closeLine
- output.write(line)
- line = triple(E55placeHolder, rdfsCoords.prefix + 'label', '\"Sede operativa\"') + closeLine
- output.write(line)
- e53placeHolder = "<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/" + row['id sede operativa'] + ">"
- line = triple(aspoPlaceHolder,
- cidocCoords.prefix + 'P74_has_current_or_former_residence',
- e53placeHolder) + closeLine
- output.write(line)
- output.write('\n')
- #
- # Limit number of entries processed (if desired)
- if (ii > max_entries):
- break
|