ASPO_CSV_to_RDF_onomastica_datini.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270
  1. #Parser to convert the Datini onomastics CSV file into TTL format
  2. # Utilities to read/write csv files
  3. import csv
  4. # Utilities to handle character encodings
  5. import unicodedata
  6. # Ordered Dicts
  7. from collections import OrderedDict
  8. import json
  9. import re
  10. # OPZIONAL IMPORTS
  11. # For timestamping/simple speed tests
  12. from datetime import datetime
  13. # Random number generator
  14. from random import *
  15. # System & command line utilities
  16. import sys
  17. # Json for the dictionary
  18. import json
  19. import_dir = '/Users/federicaspinelli/Google Drive/OVI-CNR/CSV/ASPO/ospedale/'
  20. export_dir = '/Users/federicaspinelli/Google Drive/OVI-CNR/RDF/ASPO/ospedale/'
  21. # Custom class to store URIs + related infos for the ontologies/repositories
  22. class RDFcoords:
  23. def __init__(self, uri, prefix, code = None):
  24. self.uri = uri
  25. self.prefix = prefix
  26. self.code = code
  27. # Repositories
  28. aspoCoords = RDFcoords('<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/>', 'aspo:')
  29. foafCoords = RDFcoords('<http://xmlns.com/foaf/0.1/>', 'foaf:')
  30. cidocCoords = RDFcoords('<http://www.cidoc-crm.org/cidoc-crm/>', 'crm:')
  31. schemaCoords = RDFcoords('<http://schema.org/>', 'schema:')
  32. personCoords = RDFcoords('<http://www.w3.org/ns/person#>', 'person:')
  33. nsCoords = RDFcoords('<http://www.w3.org/1999/02/22-rdf-syntax-ns#>', 'rdf:')
  34. rdfsCoords = RDFcoords('<http://www.w3.org/2000/01/rdf-schema#>', 'rdfs:')
  35. # Basic functions for triples / shortened triples in TTL format
  36. def triple(subject, predicate, object1):
  37. line = subject + ' ' + predicate + ' ' + object1
  38. return line
  39. def doublet(predicate, object1):
  40. line = ' ' + predicate + ' ' + object1
  41. return line
  42. def singlet(object1):
  43. line = ' ' + object1
  44. return line
  45. # Line endings in TTL format
  46. continueLine1 = ' ;\n'
  47. continueLine2 = ' ,\n'
  48. closeLine = ' .\n'
  49. def writeTTLHeader(output):
  50. output.write('@prefix ' + aspoCoords.prefix + ' ' + aspoCoords.uri + closeLine)
  51. output.write('@prefix ' + foafCoords.prefix + ' ' + foafCoords.uri + closeLine)
  52. output.write('@prefix ' + cidocCoords.prefix + ' ' + cidocCoords.uri + closeLine)
  53. output.write('@prefix ' + personCoords.prefix + ' ' + personCoords.uri + closeLine)
  54. output.write('@prefix ' + schemaCoords.prefix + ' ' + schemaCoords.uri + closeLine)
  55. output.write('@prefix ' + nsCoords.prefix + ' ' + nsCoords.uri + closeLine)
  56. output.write('@prefix ' + rdfsCoords.prefix + ' ' + rdfsCoords.uri + closeLine)
  57. output.write('\n')
  58. filePrefix = 'onomastica_'
  59. fileType = 'ospedale'
  60. max_entries = 1000000000
  61. with open(import_dir + filePrefix + fileType + '.csv', newline="") as csv_file, open(
  62. export_dir + filePrefix + fileType + '.ttl', 'w') as output:
  63. reader = csv.DictReader(csv_file)
  64. writeTTLHeader(output)
  65. first = True
  66. ii = 0
  67. for row in reader:
  68. # The index ii is used to process a limited number of entries for testing purposes
  69. ii = ii + 1
  70. if row['entityType'] == 'person':
  71. id_aspo = row['recordId']
  72. #placeHolders
  73. aspoPlaceHolder = aspoCoords.prefix + id_aspo
  74. line = triple(aspoPlaceHolder,
  75. nsCoords.prefix + 'type',
  76. cidocCoords.prefix + 'E21_Person') + closeLine
  77. output.write(line)
  78. line = triple(aspoPlaceHolder,
  79. nsCoords.prefix + 'type',
  80. personCoords.prefix + 'Person') + closeLine
  81. output.write(line)
  82. line = triple(aspoPlaceHolder,
  83. nsCoords.prefix + 'type',
  84. foafCoords.prefix + 'person') + closeLine
  85. output.write(line)
  86. line = triple(aspoPlaceHolder,
  87. cidocCoords.prefix + 'P1_is_identified_by',
  88. aspoPlaceHolder + "_E42") + closeLine
  89. output.write(line)
  90. line = triple(aspoPlaceHolder + "_E42",
  91. nsCoords.prefix + 'type',
  92. cidocCoords.prefix + 'E42_Identifier') + closeLine
  93. output.write(line)
  94. line = triple(aspoPlaceHolder + "_E42",
  95. rdfsCoords.prefix + 'label',
  96. '\"' + id_aspo + '\"') + closeLine
  97. output.write(line)
  98. line = triple(aspoPlaceHolder,
  99. foafCoords.prefix + 'name',
  100. '\"' + row['nameEntry@normal'] + '\"') + closeLine
  101. output.write(line)
  102. line = triple(aspoPlaceHolder,
  103. rdfsCoords.prefix + 'label',
  104. '\"' + row['nameEntry@normal'] + '\"') + closeLine
  105. output.write(line)
  106. if row['nome proprio'] != '':
  107. #Remove all white-space characters:
  108. txt = row['nome proprio']
  109. x = re.sub(" \n", "", txt)
  110. y = re.sub("\s\s", "", x)
  111. line = triple(aspoPlaceHolder,
  112. foafCoords.prefix + 'givenName',
  113. '\"' + y + '\"') + closeLine
  114. output.write(line)
  115. if row['nome di famiglia'] != '':
  116. #Remove all white-space characters:
  117. txt = row['nome di famiglia']
  118. x = re.sub("\n", " ", txt)
  119. y = re.sub("\s\s", "", x)
  120. line = triple(aspoPlaceHolder,
  121. foafCoords.prefix + 'familyName',
  122. '\"' + y + '\"') + closeLine
  123. output.write(line)
  124. if row['Alias'] != '' and row['Alias'] != ' ':
  125. #Remove all white-space characters:
  126. txt = row['Alias']
  127. x = re.sub("\n", " ", txt)
  128. y = re.sub("\s\s", "", x)
  129. line = triple(aspoPlaceHolder,
  130. schemaCoords.prefix + 'alternateName',
  131. '\"' + y + '\"') + closeLine
  132. output.write(line)
  133. if row['genere'] != '':
  134. #Remove all white-space characters:
  135. txt = row['genere']
  136. x = re.sub("\n", " ", txt)
  137. y = re.sub("\s\s", "", x)
  138. line = triple(aspoPlaceHolder,
  139. foafCoords.prefix + 'gender',
  140. '\"' + y + '\"') + closeLine
  141. output.write(line)
  142. if row['patronimico/matronimico'] != '':
  143. #Remove all white-space characters:
  144. txt = row['patronimico/matronimico']
  145. x = re.sub("\n", " ", txt)
  146. y = re.sub("\s\s", "", x)
  147. line = triple(aspoPlaceHolder,
  148. personCoords.prefix + 'patronymicName',
  149. '\"' + y + '\"') + closeLine
  150. output.write(line)
  151. if row['occupation'] != '' and row['occupation'] != ' ' :
  152. #Remove all white-space characters:
  153. txt = row['occupation']
  154. x = re.sub("\n", " ", txt)
  155. y = re.sub("\s\s", "", x)
  156. occ = re.sub(r'[^A-Za-z]','', y)
  157. occupationPlaceHolder = '<http://www.archiviodistato.prato.it/' + occ.replace(" ","_") + '>'
  158. line = triple(aspoPlaceHolder,
  159. schemaCoords.prefix + 'hasOccupation',
  160. occupationPlaceHolder) + closeLine
  161. output.write(line)
  162. line = triple(occupationPlaceHolder,
  163. nsCoords.prefix + 'type',
  164. schemaCoords.prefix + 'Occupation') + closeLine
  165. output.write(line)
  166. line = triple(occupationPlaceHolder,
  167. rdfsCoords.prefix + 'label',
  168. '\"' + y + '\"') + closeLine
  169. output.write(line)
  170. if row['avo 1'] != '':
  171. avo1 = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + id_aspo + "/avo1>"
  172. line = triple(aspoPlaceHolder,
  173. schemaCoords.prefix + 'relatedTo',
  174. avo1) + closeLine
  175. output.write(line)
  176. line = triple(avo1,
  177. nsCoords.prefix + 'type',
  178. foafCoords.prefix + 'Person') + closeLine
  179. output.write(line)
  180. line = triple(avo1,
  181. rdfsCoords.prefix + 'label',
  182. '\"' + row['avo 1'] + '\"') + closeLine
  183. output.write(line)
  184. if row['avo 2'] != '':
  185. avo2 = '<http://www.archiviodistato.prato.it/accedi-e-consulta/aspoMV001/scheda/' + id_aspo + "/avo2>"
  186. line = triple(aspoPlaceHolder,
  187. schemaCoords.prefix + 'relatedTo',
  188. avo2) + closeLine
  189. output.write(line)
  190. line = triple(avo2,
  191. nsCoords.prefix + 'type',
  192. foafCoords.prefix + 'Person') + closeLine
  193. output.write(line)
  194. line = triple(avo2,
  195. rdfsCoords.prefix + 'label',
  196. '\"' + row['avo 2'] + '\"') + closeLine
  197. output.write(line)
  198. if row['Qualifica'] != '':
  199. #Remove all white-space characters:
  200. txt = row['Qualifica']
  201. x = re.sub("\n", " ", txt)
  202. y = re.sub("\s\s", " ", x)
  203. line = triple(aspoPlaceHolder,
  204. schemaCoords.prefix + 'honorificPrefix',
  205. '\"' + y + '\"') + closeLine
  206. output.write(line)
  207. #if row['place_occupation_Qualifica'] != '':
  208. #Remove all white-space characters:
  209. # txt = row['place_occupation_Qualifica']
  210. # x = re.sub("\n", " ", txt)
  211. # y = re.sub("\s\s", "", x)
  212. # line = triple(aspoPlaceHolder,
  213. # schemaCoords.prefix + 'workLocation',
  214. # '\"' + row['place_occupation_Qualifica'].replace('\\','\\\\').replace('"','\\"') + '\"') + closeLine
  215. # output.write(line)
  216. if row['biogHist p'] != '':
  217. #Remove all white-space characters:
  218. txt = row['biogHist p']
  219. x = re.sub("\n", " ", txt)
  220. y = re.sub("\s\s", " ", x)
  221. note = re.sub("\"", "", x)
  222. line = triple(aspoPlaceHolder,
  223. cidocCoords.prefix + 'P3_has_note',
  224. '\"' + note + '\"') + closeLine
  225. output.write(line)
  226. output.write('\n')
  227. #
  228. #
  229. # Limit number of entries processed (if desired)
  230. if (ii > max_entries):
  231. break