prima del checkout di gui-devel

credo sia pronto per il merge

.gitignore

@@ -3,3 +3,4 @@
 /.pydevproject
 /org.eclipse.core.resources.prefs
 bananaSPLIT/build
+/TestFiles/

bananaSPLIT/.gitignore

@@ -1 +1,2 @@
 *.py[ocd]
+/bananaSPLIT.exe.spec

bananaSPLIT/libbananasplit/libsplit.py

@@ -3,9 +3,8 @@ Created on 2 nov 2019
 
 @author: Emanuele Trabattoni
 '''
-from libfancylogger import fancyLogger
 from slugify.slugify import slugify
-import threading, time, parse, re, copy, os
+import time, parse, re, copy, os, threading
 
 class bananaSPLITTER(threading.Thread):
 	def __init__(self, fileParams=None, logger=None):
@@ -23,6 +22,7 @@ class bananaSPLITTER(threading.Thread):
 			self.docStruct = self.fileParams['docStruct']
 			self.settings = self.fileParams['settings']
 			self.fileName = self.fileParams['name']
+			self.outPath = self.paths['OUTworkPath']+slugify(self.fileName)
 			self.beginTime = time.time()
 			pass
 		else:
@@ -31,9 +31,11 @@ class bananaSPLITTER(threading.Thread):
 	
 	def run(self):
 		self.log.info("Nuovo SPLITTER  su file: {}".format(self.fileName))
+		try:
 			self.openFile()
 			self.remEmptyLines()
 			self.splitFile()
+			
 			if self.settings['removeDuplicates']:
 				self.log.info("Controllo se ci sono dei duplicati..")
 				self.removeDuplicates()
@@ -41,15 +43,37 @@ class bananaSPLITTER(threading.Thread):
 				for idx, ff in enumerate(self.fileList):
 					ff['duplicate']=False
 					self.fileList[idx]=ff
-			print('Salto il controllo dei duplicati..')
+				self.log.warn('Salto il controllo dei duplicati..')
+			# se il parse e la rimozione dei duplicati e' andata bene
+			# preparo e inizio il salvataggio
+			if os.path.exists(self.outPath):
+				if self.settings['removeOldFiles']:
+					os.chdir(self.outPath)
+					for f in os.listdir(self.outPath):
+						os.remove(f)
+				else:
+					raise FileExistsError("Non posso sovrascrivere i vecchi file, eliminali manualmente!")
+			else:
+				os.mkdir(self.outPath)
+				os.chdir(self.outPath)
 			if self.settings['saveSeparateFiles']:
 				self.saveSeparate()
 			if self.settings['saveBodyFile']:
 				self.saveBody()
+			self.log.info("L'elaborazione del file ha richiesto {:4.2f} sec".format(time.time()-self.beginTime))
+		
+		except UnicodeDecodeError as ee:
+			self.log.critical("Il file [{}] contiene caratteri non compatibili con la codifica scelta! [{}]"
+							.format(self.fileParams['name'],ee))
+		except FileExistsError as fe:
+			self.log.critical(fe)
+		except BaseException as ee:
+			self.log.warning(ee)
 		pass
 		
 	def openFile(self):
 		try:
+			os.chdir(self.paths["INworkPath"])
 			self.log.info("Carico il contenuto..")
 			fp = open(self.fileParams['name'], mode='r', encoding=self.settings['encoding'])
 			self.rawFile = fp.readlines()
@@ -57,6 +81,7 @@ class bananaSPLITTER(threading.Thread):
 		except IOError as e:
 			self.log.critical("Impossibile aprire il file: {}! [{}]".format(self.fileName,e))
 			raise BaseException("OpenFile") 	
+			os.rmdir(self.outPath)
 		pass
 
 	def remEmptyLines(self):
@@ -77,6 +102,8 @@ class bananaSPLITTER(threading.Thread):
 	
 	def splitFile(self):  #porting del codice dal programma originale
 		self.log.info("Individuo il contenuto..")
+		self.bodyCounter=0
+		self.duplicateNumber=0
 		docNumber = 0
 		docSkipped = 0
 		docDate = {}
@@ -107,7 +134,7 @@ class bananaSPLITTER(threading.Thread):
 					# ricerco la data
 					if (lineWords[self.settings['monthPosition']]).capitalize() in self.docStruct['dateWords']:
 						try:
-							docDate=parse.parse(self.docParams['dateFormat'],l).named
+							docDate=parse.parse(self.docStruct['dateFormat'],l).named
 							docDate['month']=docDate['month'].lstrip().rstrip().capitalize()
 							docDate['month']=self.docStruct['dateWords'].index(docDate['month'])+1
 							title = ''
@@ -121,11 +148,11 @@ class bananaSPLITTER(threading.Thread):
 										newsPaperName = self.settings['nameNotFoundStr']
 								except:
 									self.log.warn("E' successo qualcosa mentre stavo cercando il nome della pubblicazione,\
-									controlla i file di uscita! \n\t[{}]".format(prevLine.strip()))
+									controlla i file di uscita! [{}]".format(prevLine.strip()))
 							else:
 								newsPaperName = self.settings['nameNotFoundStr']
 						except:
-							self.log.warn("Ho trovato una riga ambigua.. potrebbe essere una data ma non so: \n\t[{}]". format(l.strip('\r\n')))
+							self.log.warn("Ho trovato una riga ambigua.. potrebbe essere una data ma non so: [{}]". format(l.strip('\r\n')))
 							pass
 					elif lineWords[0] in self.docStruct['headWords']:
 						#cambio stato e inizializzo un nuovo documento da riempire
@@ -201,10 +228,12 @@ class bananaSPLITTER(threading.Thread):
 				ff['duplicate'] = True
 				self.fileList[idx]=ff
 				self.duplicateNumber+=1
-		self.log.info("Ho rimosso {} duplicati di {} articoli..\n". format(self.duplicateNumber, len(duplicateList)))
+		self.log.info("Ho rimosso {} duplicati di {} articoli..". format(self.duplicateNumber, len(duplicateList)))
 		pass
 		
 	def saveSeparate(self):
+		outFileCounter = 0
+		self.paths['OUTworkPath']=self.paths['OUTworkPath']+slugify(self.fileName)
 		self.log.info("Salvo gli articoli in file separati...")
 		self.log.debug("Persorso: {0}".format(self.paths['OUTworkPath'].format('nomeFile')))
 		for ff in self.fileList:
@@ -212,36 +241,36 @@ class bananaSPLITTER(threading.Thread):
 				if ff['duplicate'] == False:
 					fName=self.paths['OUTnameFormat'].format(title=slugify(ff['title'][:self.settings['maxTitleLen']]),\
 														 filename=slugify(self.fileName),\
-														 docnum=self.bodyCounter,\
+														 docnum=outFileCounter,\
 														 papername=ff['newsPaperName'].strip(),\
 														 **ff['date'])
-					out=open(self.paths['OUTworkPath']+'{0}'.format(fName),'wb')
+					out=open('{0}'.format(fName),'wb')
 					if self.settings['includeTitle']:
 						ff['content'] = ff['title']+os.linesep+ff['content']
 					out.write(ff['content'].encode(self.settings['encoding']))
 					out.close()
-					self.bodyCounter+=1
+					outFileCounter+=1
 			except IOError as e:
 				self.log.error("Qualcosa e\' andato storto, non riesco a scrivere il file: {}".format(e))
 				continue
+		if outFileCounter < self.bodyCounter:
+			raise BaseException("Ho salvato meno file rispetto a quelli trovati!")
 		pass
 	
 	def saveBody(self):
-		print('Salvo gli articoli in un singolo file vicino agli originali...')
+		self.log.info('Salvo gli articoli in un singolo file vicino agli originali...')
-		print ('Persorso: {0}'.format(self.paths['OUTworkPath'].format('nomeFile')))
+		self.log.debug('Persorso: {0}'.format(self.outPath))
+		os.chdir(self.outPath)
 		try:
 			fName=slugify(self.fileName)
-			fName='BODYFILE_{0}_{1}.txt'.format(self.fileCounter,fName[:self.settings['maxTitleLen']])
+			fName='BODYFILE_{0}.txt'.format(fName[:self.settings['maxTitleLen']])
 			fileContent = os.linesep.join([cc['content'] for cc in self.fileList])
-			out=open(self.paths['OUTworkPath']+'{0}'.format(fName),'wb')
+			out=open('{0}'.format(fName),'wb')
 			out.write(fileContent.encode(self.settings['encoding']))
 			out.close()
 		except IOError as e:
-			print("OOPS! Qualcosa e\' andato storto, non riesco a scrivere il file: {}".format(e))
+			self.log.error("Qualcosa e\' andato storto, non riesco a scrivere il file: {}".format(e))
 		pass
 
-if __name__ == "__main__":
-	logg = fancyLogger(name="LibSplit")
-	spp = bananaSPLITTER(fileParams="testfile.txt", logger=logg)
 	
 		

bananaSPLIT/libbananasplit/testEN.json

@@ -9,7 +9,7 @@
 	"name": "",
 		"paths": {
 			"INworkPath": "D:\\Emanuele\\Documenti\\workspace\\bananaSPLIT\\TestFiles\\",
-			"OUTworkPath": "D:\\Test\\Separati\\",
+			"OUTworkPath": "H:\\",
 			"OUTnameFormat": "TEST_{docnum}_{year:04d}{month:02d}{day:02d}_{title}.txt"
 		},
 		"docStruct": {
@@ -58,8 +58,8 @@
 			"nameNotFoundStr": "ND",
 			"includeTitle": true,
 			"removeDuplicates": true,
-			"showSkipped": false,
+			"showSkipped": true,
-			"showRemovedDuplicates": true,
+			"showRemovedDuplicates": false,
 			"maxTitleLen": 32,
 			"loadTXT": true,
 			"loadDOCX": false,

bananaSPLIT/libbananasplit/testITA.json

@@ -9,7 +9,7 @@
 		"name": "",
 		"paths": {
 			"INworkPath": "D:\\Emanuele\\Documenti\\workspace\\bananaSPLIT\\TestFiles\\",
-			"OUTworkPath": "D:\\Test\\Separati\\",
+			"OUTworkPath": "H:\\",
 			"OUTnameFormat": "TEST_{docnum}_{year:04d}{month:02d}{day:02d}_{title}.txt"
 		},
 		"docStruct": {

bananaSPLIT/libtestmain.py

@@ -4,23 +4,31 @@ Created on 1 dic 2019
 @author: Emanuele Trabattoni
 '''
 import os
+from glob import glob
+from copy import deepcopy
 from libsplit import bananaSPLITTER
 from libconfload import bananaCONF
 from libfancylogger import fancyLogger
 
-print("CWD-> "+os.getcwd())
+if __name__ == "__main__":
-logger = fancyLogger(fileLog = False)
+	print("CWD-> "+os.getcwd())
-confl = bananaCONF(workdir=r"./libbananasplit", logger=logger)
+	logger = fancyLogger(fileLog = False)
-confl.open()
+	confl = bananaCONF(workdir=r"./libbananasplit", logger=logger)
-
+	confl.open()
-confl.use("testEN.json")
-splconf = confl.getParams("splitter")
-splconf["name"] = splconf["paths"]["INworkPath"]+"GUARDIAN 1989.txt"
-splitter = bananaSPLITTER(fileParams=splconf, logger=logger)
-splitter.start()
-splitter.join()
-
 	
+	confl.use("testEN.json")
+	splconf = confl.getParams("splitter")
+	splist = []
+	os.chdir(splconf["paths"]["INworkPath"])
+	for f in glob("*.txt"):
+		splconf["name"] = f
+		logger.info("-"*80)
+		splitter = bananaSPLITTER(fileParams=deepcopy(splconf), logger=logger)
+		splist.append(splitter)
+		splitter.start()
+		splitter.join()
+
+	logger.info("\n"+"="*50+"\n\tFINITO!!!\n"+"="*50)