# IMPORT ESSENZIALI

# Per il parsing dell'XML -- questo pacchetto è incluso anche nel più generale lxml
import xml.etree.ElementTree as ET
# Utilities per leggere/scrivere files csv
import csv
# Utilities per gestire i character encodings
import unicodedata
# Dizionari ordinati
from collections import OrderedDict


# IMPORT OPZIONALI

# Per fare un stima della velocità delle varie istruzioni
from datetime import datetime
# Generatore di numeri casuali -- può sempre servire in fase di testing
from random import *
# Può servire per alcuni test
import sys

# La funzione BASE: traceElems
def traceElems(node: ET.Element, condition, parents: list = [], coords: list = []):
    res = []
    jj = 0
    for child in node:
        if condition(child):
            res.append({'a_par': parents+[node],
                        'coords': coords+[jj], 'child': child})
        else:
            res = res + traceElems(child, condition, parents+[node], coords+[jj])
        jj = jj+1   
    return res

# Funzione-base per stoppare traceElems
def isLeafOrC(aa: ET.Element):
    if(aa.tag=='c' or len(aa)==0):
        return True
    else:
        return False

# Funzioni-utilità che servono solo a visualizzare meglio i dati sul notebook.
def shownode(node: ET.Element):
    return (node.tag, node.attrib, node.text.replace('\t','').replace('n','').strip() \
                               if type(node.text) is str else '')

def shownodelist(el: ET.Element):
    return list(map(shownode, el))


# Utility copiata da INTERNEZZ -- versione 'multipla' del metodo str.index:
def indices(lst, element):
    result = []
    offset = -1
    while True:
        try:
            offset = lst.index(element, offset+1)
        except ValueError:
            return result
        result.append(offset)

import_dir = '/Users/federicaspinelli/Google Drive/OVI:CNR/LAVORO 2020/SELEZIONE CONTENUTI/01_ASPO/XDAMS/'
export_dir = '/Users/federicaspinelli/Google Drive/OVI:CNR/CSV/ASPO/ospedale/'

ts1 = datetime.timestamp(datetime.now())

treeDatini = ET.parse(import_dir + 'export_aspoSt005--ospedale.xml')
rootDatini = treeDatini.getroot()

ts2 = datetime.timestamp(datetime.now())
print(ts2 - ts1)

cLevs = set(map(lambda a : a.attrib['level'], rootDatini.iter('c')))
print(cLevs)

ts1 = datetime.timestamp(datetime.now())

allCs = {}

for label in cLevs:
    def tempFilt(aa: ET.Element):
        if(aa.tag=='c' and aa.attrib['level']==label):
            return True
        else:
            return False
       
    allCs[label] = traceElems(rootDatini, tempFilt);
    print('# di tag "c", livello ' + label + ', primo passaggio:', len(allCs[label]))

print()
print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

allCs2 = {}

for label in cLevs:
    partial = allCs[label]
    print('# di tag "c", livello ' + label + ', primo passaggio:', len(partial))
    allCs2[label] = partial
    partialUpdate = []
    while True:
        def tempFilt(aa: ET.Element):
            if(aa.tag=='c' and aa.attrib['level']==label):
                 return True
            else:
                 return False
        for node in partial:
            partialUpdate = partialUpdate + traceElems(node['child'], tempFilt)
        #print(len(partialUpdate))
        partial = partialUpdate
        if(len(partialUpdate)==0):
            break
        allCs2[label] = allCs2[label] + partial
        partialUpdate = []

    print('# di tag "c", livello ' + label + ', totali:', len(allCs2[label]))

print()
print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ii = 1
level = 'otherlevel'
test = allCs2[level][ii]
toProc = traceElems(test['child'], isLeafOrC)

# Commentare/scommentare per stampare qui / su file
# (vedi anche in fondo alla cella)
#provaFileName = 'out.txt'
#orig_stdout = sys.stdout
#fp = open(export_dir + provaFileName, 'w')
#sys.stdout = fp
# fino qui + in fondo

print()
print()
print('Level:', level)
print('#:', ii)
print()
for node in toProc:
    print(shownodelist(node['a_par']))
    print(node['coords'])
    print(shownode(node['child']))
    print('# of children:', len(node['child']))
    print()


# Commentare/scommentare per stampare qui / su file
#sys.stdout = orig_stdout
#fp.close()

def traduttoreItem(elem):
    # Variabile che contiene l'output della traduzione:
    csvProt = {}

    # Processo i nodi-parent di 'elem'
    par_tags = list(map(lambda a: a.tag, elem['a_par']))
    par_attributes = list(map(lambda a: a.attrib, elem['a_par']))
    
    # e0: Le varie id dei nodi parent
    for ii in indices(par_tags, 'c'):
        key = 'id_' + par_attributes[ii]['level']
        csvProt[key] = par_attributes[ii]['id']

    # Processo i nodi-child di 'elem'
    toProc = traceElems(elem['child'], isLeafOrC)
    first = True
    for node in toProc:
        tags = list(map(lambda a: a.tag, node['a_par'])) + [node['child'].tag]
        attributes = list(map(lambda a: a.attrib, node['a_par'])) + [node['child'].attrib]
        content = node['child'].text

        # Da controllare solo per il primo nodo
        # (informazioni a livello del nodo, uguali per tutti i figli)
        if(first):
            # e1 ID della item
            csvProt['id'] = attributes[tags.index('c')]['id']
            # e2 Audience: external o internal
            try:
                csvProt['audience'] = attributes[tags.index('c')]['audience']
            except:
                pass
            # e3 Otherlevel
            try:
                csvProt['altro_livello'] = attributes[tags.index('c')]['otherlevel']
            except:
                pass
            first = False

        # La 'ciccia': si processa il contenuto vero e proprio
        # e4 Repository (qui dovrebbe essere sempre l'Archivio di Prato)
        if('repository' in tags):
            csvProt['repository'] = content  

        # e8 Tipologia
        try:
            ii = tags.index('materialspec')
            if(attributes[ii]['label']=='tipologia'): 
                csvProt['tipologia'] = content
        except:
            pass

        # e9 Segnatura attuale
        try:
            ii = tags.index('num')
            type1 = attributes[ii]['type']
            if(type1=='nuovo ordinamento'):
               csvProt['segnatura_attuale'] = content
        except:
            pass
        # e9 Segnatura precedente (Odd)
        if('odd' in tags):
            csvProt['segnatura_precedente'] = content       
        
        # e11 Il titolo da unittitle
        try:
            aa = csvProt['titolo_aspo']
        except:
            try:
                ii = tags.index('unittitle')
                try:
                    csvProt['titolo_aspo'] = str(node['a_par'][ii].text).replace('\t','').replace('\n','').strip()
                except:
                    csvProt['titolo_aspo'] = str(content).replace('\t','').replace('\n','').strip()
            except:
                pass
        
        # e12 Scope-content head & body
        if('scopecontent' in tags):
            if('p' in tags):
                csvProt['scope-content_body'] = content

        # e14 Nome della compagnia
        try:
            ii = tags.index('corpname')
            if(attributes[ii]['authfilenumber']):
                try:
                    authId = attributes[ii]['authfilenumber']
                    csvProt['compagnia'] = '{"nome": ' + "\"" + content + "\"" + ', "authID": ' + "\"" + authId + "\"" + '}'
                except:
                    csvProt['compagnia'] = '{"nome": ' + "\"" + content + "\"" + '}'
        except:
            pass
        
        # e16 Persona
        if ('controlaccess' in tags):
            try: 
                ii=tags.index('persname')       
                key='persona'  
                authId = attributes[ii]['authfilenumber']
                try:       
                    csvProt[key] = csvProt[key] + ' | {"nome": ' + "\"" + content + "\"" + ', "authID": ' + "\"" + authId + "\"" +'}'
                except:                         
                    csvProt[key] = '{"nome": ' + "\"" + content + "\"" + ', "authID": ' + "\"" + authId + "\"" +'}'
            except:                
                try:
                    csvProt[key] = csvProt[key] + ' | {"nome": ' + "\"" + content + "\"" + '}'
                except:
                    csvProt[key] = '{"nome": ' + "\"" + content + "\"" + '}'

        # e17 Date
        if ('unittitle' in tags):
            try:
                ii = tags.index('date')
                key = 'data'
                csvProt[key] = content
            except:
                pass
       
        # e18 Data 1: periodo
        if('unitdate' in tags):
            csvProt['data_periodo'] = content
        
        # e20 Supporto fisico
        try:
            ii = tags.index('physfacet')
            if(attributes[ii]['type']=='supporto'):
                csvProt['supporto'] = content
        except:
            pass

        # e21 Physdesc 
        if('extent' in tags):
            csvProt['numero'] = content
        if('genreform' in tags):
            csvProt['genere'] = content
        
        # e21 Dimensions
        try:
            ii = tags.index('dimensions')
            try:
                csvProt['dimensione_altezza_larghezza_spessore'] = csvProt['dimensione_altezza_larghezza_spessore'] + ' | ' + content
            except:
                csvProt['dimensione_altezza_larghezza_spessore'] = content
        except:
            pass
        
        # e22 Phystech  
        try:
            ii = tags.index('phystech')
            try:
                csvProt['conservazione'] = csvProt['conservazione'] + ' | ' + content
            except:
                csvProt['conservazione'] = content
        except:
            pass
        
        # e24 Note
        if('note' in tags):
            csvProt['nota'] = content
        
        # e26 Oggetto digitale allegato (nome)
        try:
            ii = tags.index('daoloc')
            out = attributes[ii]['title']
            try:
                csvProt['oggetto_digitale'] = csvProt['oggetto_digitale'] + ' | ' + out
            except:
                csvProt['oggetto_digitale'] = out
        except:
            pass

    return csvProt


# Di pari passo alla funzione, definisco un dict contenente tutti gli header;
# servirà per il CSV.
itemHeader = OrderedDict()

# e1 ID dell'entità
itemHeader.update({'id': '<c level="X" id=#>'})

# e2 Audience: external o internal
itemHeader.update({'audience': '<c level="#" audience=#>'})

# e3 Otherlevel
itemHeader.update({'altro_livello': '<c otherlevel=#>'})

# e4 Repository (qui dovrebbe essere sempre l'Archivio di Prato)
itemHeader.update({'repository': '<repository>#'})

# e8 Tipologia
itemHeader.update({'tipologia': '<materialspec label="tipologia">#'})

# e9 Segnatura attuale
itemHeader.update({'segnatura_attuale': '<num type="nuovo ordinamento">#'})

# e9 Segnatura precedente
itemHeader.update({'segnatura_precedente': '<odd>#'})

# e11 Titolo ASPO
itemHeader.update({'titolo_aspo': '<unittitle>#'})

# e12 Scope content, head & body
itemHeader.update(
{'scope-content_head': '<scopecontent><head>#',
 'scope-content_body': '<scopecontent><p>#'})

# e14 Nome della compagnia
itemHeader.update({'compagnia': '<corpname>#'})

# e15 Soggetto
itemHeader.update({'soggetto': '<subject>#'})

# e16 Persona
itemHeader.update({'persona': '<persname authfilenumber=#>#'})

# e17 Date
itemHeader.update(
{'data': '<date>#'})

# e18 Data 1: periodo
itemHeader.update({'data_periodo': '<unitdate>#'})

# e20 Supporto fisico
itemHeader.update({'supporto': '<physfacet type="supporto">#'})

# e21 descrizione fisica
itemHeader.update({'numero': '<extent>#'})
itemHeader.update({'genere': '<genreform>#'})
# e21 dimensions
itemHeader.update({'dimensione_altezza_larghezza_spessore': '<dimensions>#'})

# e22 Phystech
itemHeader.update({'conservazione': '<phystech>#'})

# e23 Consistenza
itemHeader.update({'consistenza': '<extent unit=#1>#2, #1: #2'})

# 24 Note
itemHeader.update({'nota': '<note>#'})

# e26 Oggetto digitale allegato (nome)
itemHeader.update({'oggetto_digitale': '<daoloc title=#>'})

#0: Le varie id dei nodi parent
itemHeader.update(
{'id_subfonds': '<c level="subfonds" id=#>',
 'id_fonds': '<c level="fonds" id=#>',
 'id_series': '<c level="series" id=#>',
 'id_subseries': '<c level="subseries" id=#>',
 'id_recordgrp': '<c level="recordgrp" id=#>',
 'id_otherlevel': '<c level="otherlevel" id=# otherlevel="subfile">',
 'id_collection': '<c level="collection" id=#>',
 'id_subgrp': '<c level="subgrp" id=#>',
 'id_collection': '<c level="collection" id=#>',
 'id_file': '<c level="file" id=#>'})

test = allCs2['file'][0]
toShow = traduttoreItem(test)
for key in toShow.keys():
    print(key + ': ' + str(toShow[key]))
    print()

# Do it! Export del CSV - items.

ts1 = datetime.timestamp(datetime.now())

# Apro il file per l'export
with open(export_dir + "data_file.csv", "w", newline="") as csv_file:
    # Definisco la classe-motore per l'export
    writer = csv.DictWriter(csv_file, fieldnames=list(itemHeader.keys()))
    # Scrivo l'intestazione
    writer.writeheader()
    # Scrivo la seconda riga, esplicativa
    writer.writerow(itemHeader)
    # Scrivo gli item tradotti, uno a uno
    for ii in range(len(allCs2['file'])):
        test = allCs2['file'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

subSeriesKeys = set()
for ii in range(len(allCs2['subgrp'])):
    test = allCs2['subgrp'][ii]
    subSeriesKeys = subSeriesKeys.union( traduttoreItem(test).keys() )

subSeriesHeader = OrderedDict()
for key in itemHeader:
    if(key in subSeriesKeys):
        subSeriesHeader[key] = itemHeader[key]


with open(export_dir + "data_subgrp.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(subSeriesHeader.keys()))
    writer.writeheader()
    writer.writerow(subSeriesHeader)
    for ii in range(len(allCs2['subgrp'])):
        test = allCs2['subgrp'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

subSeriesKeys = set()
for ii in range(len(allCs2['subseries'])):
    test = allCs2['subseries'][ii]
    subSeriesKeys = subSeriesKeys.union( traduttoreItem(test).keys() )

subSeriesHeader = OrderedDict()
for key in itemHeader:
    if(key in subSeriesKeys):
        subSeriesHeader[key] = itemHeader[key]


with open(export_dir + "data_subseries.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(subSeriesHeader.keys()))
    writer.writeheader()
    writer.writerow(subSeriesHeader)
    for ii in range(len(allCs2['subseries'])):
        test = allCs2['subseries'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

seriesKeys = set()
for ii in range(len(allCs2['series'])):
    test = allCs2['series'][ii]
    seriesKeys = seriesKeys.union( traduttoreItem(test).keys() )

seriesHeader = OrderedDict()
for key in itemHeader:
    if(key in seriesKeys):
        seriesHeader[key] = itemHeader[key]


with open(export_dir + "data_series.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(seriesHeader.keys()))
    writer.writeheader()
    writer.writerow(seriesHeader)
    for ii in range(len(allCs2['series'])):
        test = allCs2['series'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

subfondsKeys = set()
for ii in range(len(allCs2['subfonds'])):
    test = allCs2['subfonds'][ii]
    subfondsKeys = subfondsKeys.union( traduttoreItem(test).keys() )

subfondsHeader = OrderedDict()
for key in itemHeader:
    if(key in subfondsKeys):
        subfondsHeader[key] = itemHeader[key]


with open(export_dir + "data_subfonds.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(subfondsHeader.keys()))
    writer.writeheader()
    writer.writerow(subfondsHeader)
    for ii in range(len(allCs2['subfonds'])):
        test = allCs2['subfonds'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

fondsKeys = set()
for ii in range(len(allCs2['fonds'])):
    test = allCs2['fonds'][ii]
    fondsKeys = fondsKeys.union( traduttoreItem(test).keys() )

fondsHeader = OrderedDict()
for key in itemHeader:
    if(key in fondsKeys):
        fondsHeader[key] = itemHeader[key]


with open(export_dir + "data_fonds.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(fondsHeader.keys()))
    writer.writeheader()
    writer.writerow(fondsHeader)
    for ii in range(len(allCs2['fonds'])):
        test = allCs2['fonds'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

collectionKeys = set()
for ii in range(len(allCs2['collection'])):
    test = allCs2['collection'][ii]
    collectionKeys = collectionKeys.union( traduttoreItem(test).keys() )

collectionHeader = OrderedDict()
for key in itemHeader:
    if(key in collectionKeys):
        collectionHeader[key] = itemHeader[key]


with open(export_dir + "data_collection.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(collectionHeader.keys()))
    writer.writeheader()
    writer.writerow(collectionHeader)
    for ii in range(len(allCs2['collection'])):
        test = allCs2['collection'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

subSeriesKeys = set()
for ii in range(len(allCs2['recordgrp'])):
    test = allCs2['recordgrp'][ii]
    subSeriesKeys = subSeriesKeys.union( traduttoreItem(test).keys() )

subSeriesHeader = OrderedDict()
for key in itemHeader:
    if(key in subSeriesKeys):
        subSeriesHeader[key] = itemHeader[key]


with open(export_dir + "data_recordgrp.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(subSeriesHeader.keys()))
    writer.writeheader()
    writer.writerow(subSeriesHeader)
    for ii in range(len(allCs2['recordgrp'])):
        test = allCs2['recordgrp'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

ts1 = datetime.timestamp(datetime.now())

otherlevelKeys = set()
for ii in range(len(allCs2['otherlevel'])):
    test = allCs2['otherlevel'][ii]
    otherlevelKeys = otherlevelKeys.union( traduttoreItem(test).keys() )

otherlevelHeader = OrderedDict()
for key in itemHeader:
    if(key in otherlevelKeys):
        otherlevelHeader[key] = itemHeader[key]


with open(export_dir + "data_otherlevel.csv", "w", newline="") as csv_file:
    writer = csv.DictWriter(csv_file, fieldnames=list(otherlevelHeader.keys()))
    writer.writeheader()
    writer.writerow(otherlevelHeader)
    for ii in range(len(allCs2['otherlevel'])):
        test = allCs2['otherlevel'][ii]
        writer.writerow(traduttoreItem(test))

print('Tempo trascorso:', datetime.timestamp(datetime.now()) - ts1)

otherlevelKeys = set()
for ii in range(len(allCs2['otherlevel'])):
    test = allCs2['otherlevel'][ii]
    otherlevelKeys = otherlevelKeys.union( traduttoreItem(test).keys() )

otherlevelKeys

otherlevelHeader = OrderedDict()
for key in itemHeader:
    if(key in otherlevelKeys):
        otherlevelHeader[key] = itemHeader[key]

otherlevelHeader

itemHeader.keys()

otherlevelHeader

itemHeader