archive-file

#!/usr/bin/env python3

"""

Upload file to Noark 5 API for archiving.

"""

from __future__ import print_function

__license__ = 'GNU General Public License v2 or later at users choice'
__author__ = 'Petter Reinholdtsen'

import sys
import os
sys.path.append(os.path.join(sys.path[0],'lib'))

import argparse
import json
import email
import magic
from hashlib import sha256
import n5core.endpoint
from n5core.pick import pickUnlessOne
try:
    # Python 3
    from PyPDF2 import PdfFileReader
except:
    from pyPdf import PdfFileReader

class infoReader():
    def __init__(self, fh, filepath):
        self.title = None
        self.filename = os.path.basename(filepath)
    def getTitle(self): return self.title
    def getFilename(self): return self.filename

class pdfReader(infoReader):
    def __init__(self, fh, filepath):
        """
Extract document information from PDF.  Must read every value in the
constructor, to avoid moving the file handle pointer for others later.
"""
        infoReader.__init__(self, fh, filepath)
        reader = PdfFileReader(fh)
        self.title = reader.getDocumentInfo().title
        self.producer = reader.getDocumentInfo().producer
        self.creator = reader.getDocumentInfo().creator
        #reader.getDocumentInfo().creationdate ?
        print(self.title)

class imageReader(infoReader):
    def __init__(self, fh, filepath):
        infoReader.__init__(self, fh, filepath)
        return

class rfc822Reader(infoReader):
    def __init__(self, fh, filepath):
        infoReader.__init__(self, fh, filepath)
        self.filename = None
        fh.seek(0, os.SEEK_SET)
        msg = email.message_from_file(fh)
        if 'subject' in msg:
            subject, encoding = \
                email.Header.decode_header(msg.get('subject'))[0]
            if encoding!=None:
                self.title = subject.decode(encoding)
            print("S: %s" % self.title)
        if 'message-id' in msg:
            self.filename = msg['message-id']
        return

# Mapping from magic string to Noark 5 format info
# Based on the spec list of formats on page 173.
formatlist = {
    # PDF/A - ISO 19005 1:205
    'application/pdf': {
        'inforeader': pdfReader,
#        'format':     { 'kode': 'RA-PDF' },
        },
    # JPEG (ISO 10918-1:1994)
    'image/jpeg': {
        'inforeader': imageReader,
#        'format':     { 'kode': 'RA-JPEG' },
    },
    # TIFF versjon 6
    'image/tiff': {
        'inforeader': imageReader,
#        'format':     { 'kode': 'RA-TIFF6' },
    },
    # Ren text
    # XML v1
    # SOSI v2.2 (1995 el. nyere)
    # MPEG-2 (ISO 13818-2)
    # MP3 (ISO 11172-3 PCM eller PCM-basert)

    # Unofficial types:
    'message/rfc822': {
        'inforeader': rfc822Reader,
#        'format':     { 'kode': 'RFC822' },
    },
}

def objectOperation(json, relation):
    """
Return the URL/href for a given relation on the object represented by
the json structure.

"""
    if json is None or '_links' not in json:
        return None
    if relation in json['_links']:
        return json['_links'][relation]['href']
    return None

def createObject(api, url, data):
        (c, res) = api.json_post(url, data)
        fondinfo = json.loads(c)
        return fondinfo

def getFileSize(filehandle):
    curpos = filehandle.tell()
    filehandle.seek(0, os.SEEK_END)
    size = filehandle.tell()
    filehandle.seek(curpos, os.SEEK_SET)
    return size

def main():
    relbase = 'https://rel.arkivverket.no/noark5/v5/api'
    baseurl = "https://localhost:8092/noark5v5/"
    parser = argparse.ArgumentParser()
    parser.add_argument("--baseurl", help="(default is %s)" % baseurl)
    parser.add_argument("--verbose", help="print more debug information",
                        action="store_true")
    parser.add_argument("filename", nargs='+', help="file to archive")
    args = parser.parse_args()

    if args.baseurl:
        baseurl = args.baseurl

    retval = 0
    api = n5core.endpoint.Endpoint(baseurl)
    api.verbose = args.verbose
    api.login()

    fondsrel = relbase + '/arkivstruktur/arkiv/'
    fondshref = api.findRelation(fondsrel)
    
    (c, res) = api.json_get(fondshref)
    fondsinfo = json.loads(c)
    info = pickUnlessOne(fondsinfo, 'arkiv')
    if info is None:
        print("error: Fant intet relasjon for %s.  Tom database?" % fondsrel)
        print("error: Kan ikke laste opp dokumenter uten definert arkiv.")
        return

    seriesrel = relbase + '/arkivstruktur/arkivdel/'
    serieshref = objectOperation(info, seriesrel)
    if serieshref is None:
        print("error: Fant ingen relasjon for %s.  Tom database?" % seriesrel)
        print("error: Kan ikke laste opp dokumenter uten definert arkivdel.")
        return
    (c, res) = api.json_get(serieshref)
    seriesinfo = json.loads(c)
    info = pickUnlessOne(seriesinfo, 'arkivdel')
    if info is None:
        print("error: Fant ingen arkivdel.  Tom database?")
        print("error: Kan ikke laste opp dokumenter uten definert arkivdel.")
        return

    filesrel = relbase + '/arkivstruktur/mappe/'
    fileshref = objectOperation(info, filesrel)
    if fileshref is None:
        print("error: Fant intet relasjon for %s.  Tom database?" % filesrel)
        print("error: Kan ikke laste opp dokumenter uten definert mappe.")
        return
    (c, res) = api.json_get(fileshref)
    filesinfo = json.loads(c)
    fileinfo = pickUnlessOne(filesinfo, 'mappe')
    if fileinfo is None:
        print("error: Fant ingen mappe.  Tom database?")
        print("error: Kan ikke laste opp dokumenter uten definert mappe.")
        return

    # Test if it is possible to reach a dokumentobject by following
    # the relations from the top.
    if False:
        fileshref = objectOperation(fileinfo, relbase + '/arkivstruktur/registrering/')
        #print("Registrering %s" % fileshref)
        (c, res) = api.json_get(fileshref)
        filesinfo = json.loads(c)
        reginfo = pickUnlessOne(filesinfo, 'registrering')

        fileshref = objectOperation(reginfo, relbase + '/arkivstruktur/dokumentbeskrivelse/')
        #print("Dokumentbeskrivelse %s" % fileshref)
        (c, res) = api.json_get(fileshref)
        filesinfo = json.loads(c)
        docdescinfo = pickUnlessOne(filesinfo, 'dokumentbeskrivelse')

        fileshref = objectOperation(docdescinfo, relbase + '/arkivstruktur/dokumentobjekt/')
        #print("Dokumentobjekt %s" % fileshref)
        (c, res) = api.json_get(fileshref)
        filesinfo = json.loads(c)
        docobjinfo = pickUnlessOne(filesinfo, 'dokumentobjekt')

    #
    # Correct file located, add record, document description and
    # document object for each file.
    #
    for filename in (args.filename):
        format = None
        mimetype = None
        inforeader = None
        fh = file(filename, "rb")

        hashalg = "SHA-256"
        hasher = sha256()
        for chunk in iter(lambda: fh.read(8192), b''): 
            hasher.update(chunk)
        hash = hasher.hexdigest()

        mimetype = magic.from_file(filename, mime=True)
        if mimetype in formatlist:
            inforeader = formatlist[mimetype]['inforeader']
            if 'format' in formatlist[mimetype]:
                format = formatlist[mimetype]['format']
        if mimetype is None:
            print("error: Unknown mime type '%s', skipping" % mimetype)
            continue
        
        filesize = getFileSize(fh)
        r = inforeader(fh, filename)
        doctitle = r.getTitle()
        basename = r.getFilename()
        if doctitle is None or '' == doctitle:
            doctitle = basename
        variantformatcode = 'A'
        variantformat = "Arkivformat"
        fh.seek(0, os.SEEK_SET)

        print("Uploading %s\n  Document title: %s\n  File %s: %s" % \
              (filename, doctitle, fileinfo['systemID'], fileinfo['tittel']))
        
        newreghref = objectOperation(fileinfo, relbase + '/arkivstruktur/ny-registrering/')
        recordinfo = createObject(api, newreghref, { 'tittel': doctitle })

        # adding documentbeskrivelse between registrering av
        # dokumentobject might be optional, but skipping it is not
        # implemented by Nikita 2017-03-22.  Once it is implemented,
        # consier replacing 'docdescifo' with 'recordinfo' in the
        # ny-dokumentobject call to objectOperation().
        newdocdeschref = objectOperation(recordinfo,
                                         relbase + '/arkivstruktur/ny-dokumentbeskrivelse/')
        docdescinfo = createObject(api, newdocdeschref, {
            'tittel' : doctitle,
            'dokumenttype': {
                'kode': 'B',
                'kodenavn': 'Brev'
            },
            'dokumentstatus': {
                'kode': 'F',
                'kodenavn': 'Dokumentet er ferdigstilt'
            },
#            'dokumentnummer': str(1),
            'tilknyttetRegistreringSom': {
                'kode': 'H',
                'kodenavn': 'Hoveddokument'
            },
        })

        newdocobjhref = objectOperation(docdescinfo,
                                        relbase + '/arkivstruktur/ny-dokumentobjekt/')
        data = {
            "versjonsnummer" : 0,
            "variantformat"  : {
                'kode': variantformatcode,
                'kodenavn': variantformat,
            },
            "mimeType"       : mimetype,
            "filstoerrelse"  : str(filesize),
            "filnavn"        : basename,
            "sjekksum"       : hash,
            "sjekksumAlgoritme" : hashalg,
        }
        if format:
            data['format'] = format
        docobjinfo = createObject(api, newdocobjhref, data)

        newfilehref = objectOperation(docobjinfo, relbase + '/arkivstruktur/fil/')
        (c, res) = api.post(newfilehref, fh, mimetype, length=filesize)
        if 201 != res.code:
            print("error: upload failed (status == %d)" % res.code)
            retval = 1
    return retval

if __name__ == '__main__':
    exit(main())