From 4824202f591bd480627a498c867f659640a21ac7 Mon Sep 17 00:00:00 2001
From: seefo <robbie.uvanni@gmail.com>
Date: Sat, 26 May 2018 07:42:04 -0400
Subject: [PATCH] Fixed halting issues in python dsrc build script

---
 utils/build_dsrc.py | 56 +++++++++++++++++++++++++++++++++------------
 1 file changed, 42 insertions(+), 14 deletions(-)

diff --git a/utils/build_dsrc.py b/utils/build_dsrc.py
index 1610792..9e3795f 100755
--- a/utils/build_dsrc.py
+++ b/utils/build_dsrc.py
@@ -10,11 +10,13 @@ import datetime
 import copy
 
 q = queue.Queue()
+completedFileCount = 0
+completedFiles = []
 failedTasks = []
 lastTickStatus = lambda: None
 
-def processQueue():
-	while q.empty() == False:
+def displayQueueStatus():
+	while q.qsize() > 0:
 		completedTasks = totalTaskCount - q.qsize()
 		progressPercent = (float(completedTasks) / float(totalTaskCount)) * 100.0
 		
@@ -37,7 +39,14 @@ def processQueue():
 		time.sleep(1)
 	q.join()
 
-def processTask( toolCmd, srcFile, dstFile ):
+def processTask( task ):
+	toolCmd = task[0]
+	srcFile = task[1]
+	dstFile = task[2]
+
+	if srcFile in completedFiles:
+		return
+
 	# remove dst file so we can validate successful creation
 	if os.path.isfile(dstFile):
 		os.remove(dstFile)
@@ -51,6 +60,7 @@ def processTask( toolCmd, srcFile, dstFile ):
 		if not os.path.exists(dstFolder):
 			os.makedirs(dstFolder)
 	except:
+		print 'exception'
 		pass
 
 	if displayCmdOutput:
@@ -61,14 +71,20 @@ def processTask( toolCmd, srcFile, dstFile ):
 
 	# validate dst file was successfully created and no errorcode
 	if os.path.isfile(dstFile) is False or retcode is not 0:
-		failedTasks.append([toolCmd, srcFile, dstFile])
+		failedTasks.append(task)
+	else:
+		global completedFileCount
+		completedFileCount += 1
+
+	completedFiles.append(srcFile)
 
 def queueWorker():
-	while q.empty() == False:
+	while True:
 		task = q.get()
-		processTask(task[0], task[1], task[2])
+		processTask(task)
 		q.task_done()
 
+
 def startQueueWorkers():
 	for i in range(numberOfWorkerThreads):
 		t = threading.Thread(target=queueWorker)
@@ -83,7 +99,7 @@ def walkAndCompareAndRun( src, dst, srcExt, dstExt, runCmd ):
 			file = "%s/%s" % (root, f)
 			ext = os.path.splitext(file)[1]
 
-			if(ext == srcExt):
+			if ext == srcExt and not file in ignoredFiles:
 				dstFile = file.replace(src, dst).replace(srcExt, dstExt)
 
 				if os.path.isfile(dstFile): 
@@ -104,6 +120,12 @@ lastTickSampleRate = 10     # sample rate in seconds, for eta/speed
 buildAllFiles = False       # enable to skip modification time comparison
 displayCmdOutput = False    # enable to display output of tools
 
+ignoredFiles = [
+	"./dsrc/sku.0/sys.shared/built/game/misc/quest_crc_string_table.tab",
+	"./dsrc/sku.0/sys.server/built/game/misc/object_template_crc_string_table.tab",
+	"./dsrc/sku.0/sys.client/built/game/misc/object_template_crc_string_table.tab"
+]
+
 # build datatables
 walkAndCompareAndRun( "./dsrc/sku.0/", 
 					  "./data/sku.0/", 
@@ -123,7 +145,7 @@ walkAndCompareAndRun( "./dsrc/sku.0/sys.server/compiled/game/script",
 					  "./data/sku.0/sys.server/compiled/game/script", 
 					  ".java", 
 					  ".class", 
-					  "javac -Xlint:-options -encoding utf8 -classpath \"data/sku.0/sys.server/compiled/game\" -d \"data/sku.0/sys.server/compiled/game\" -sourcepath \"dsrc/sku.0/sys.server/compiled/game\" -g -deprecation $srcFile")
+					  "javac -Xlint:-options -encoding utf8 -classpath data/sku.0/sys.server/compiled/game -d data/sku.0/sys.server/compiled/game -sourcepath dsrc/sku.0/sys.server/compiled/game -g -deprecation $srcFile")
 
 # prepare to start working
 totalTaskCount = q.qsize()
@@ -135,7 +157,7 @@ lastTickStatus.completedTasks = 0
 lastTickStatus.time = start 
 
 # start processing our queue of build tasks while there are tasks
-processQueue()
+displayQueueStatus()
 
 # reprocess failed tasks incase it was the result of a dependency issue
 if len(failedTasks) > 0:
@@ -143,18 +165,24 @@ if len(failedTasks) > 0:
 
 	while previousFailedTaskCount > len(failedTasks):
 		previousFailedTaskCount = len(failedTasks)
-		for task in failedTasks: q.put(task)
-		# attempt next pass-through
+		failedTasksCopy = copy.deepcopy(failedTasks)
+
+		completedFiles = []
 		failedTasks = []
-		processQueue()
+
+		for task in failedTasksCopy:
+			q.put(task)
+
+		displayQueueStatus()
 
 	# no more files were built so prepare for final output
 	if len(failedTasks) == previousFailedTaskCount: print ""
 
+
 # display our final results
 end = time.time()
-print "[***] Successfully built %d files in %d seconds" % (totalTaskCount - len(failedTasks), end - start)
+print "[***] Successfully built %d files in %d seconds" % (completedFileCount, end - start)
 
 # display results
 print "[***] Failed to build %d tasks" % (len(failedTasks))
-for task in failedTasks: print "FAILED TO BUILD %s" % (task[1])
\ No newline at end of file
+for task in failedTasks: print "FAILED TO BUILD %s" % (task[1])