Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit bb6c297b authored by Davide Lagoa's avatar Davide Lagoa
Browse files

add data update method

parent 10ad0761
.idea*
.test*
code/tests.py
\ No newline at end of file
......@@ -79,8 +79,30 @@ def removeFromProcessingAndResults(output_path, processingPath, exit_code):
if os.path.exists(processingPath):
shutil.rmtree(processingPath)
def update_data():
base_url = 'https://merlin-sysbio.org/data/transyt/scraper/'
downloads_dir = '/transyt_downloads/'
file_dir = downloads_dir + 'workdir.tar.gz'
if os.path.exists(downloads_dir):
os.system("rm -r -f " + downloads_dir)
os.mkdir(downloads_dir)
os.system("wget -q " + base_url + "workdir.tar.gz -O " + file_dir + " &&"
" tar -xf " + file_dir +" -C " + downloads_dir + " && "
" rm -r -f /workdir/scraper && mv " + downloads_dir + "/workdir/scraper /workdir/scraper")
if __name__ == "__main__":
processingPath = sys.argv[1]
resultsPath = sys.argv[2]
transyt(processingPath, resultsPath)
#1 - update data
#n - run classification
if "1" == sys.argv[3]:
update_data()
else:
processingPath = sys.argv[1]
resultsPath = sys.argv[2]
transyt(processingPath, resultsPath)
......@@ -7,6 +7,9 @@ import FilesUtilities
import logging
from logging.handlers import TimedRotatingFileHandler
DATA_BASE_URL = 'https://merlin-sysbio.org/data/transyt/scraper/'
CURRENT_MD5 = None
app = Flask(__name__)
logPath = '/workdir/logs/workerlogs'
......@@ -28,8 +31,6 @@ logger.setLevel(logging.DEBUG)
PROCESS_PATH = '/home/transyt.py'
SUBMISSIONS_PATH = '/workdir/workerSubmissions/'
RESULTS_PATH = '/workdir/resultsWorker/'
#RESULTS_PATH = "C:\\Users\\BioSystems\\Desktop\\testFolder\\resultsWorker\\"
#SUBMISSIONS_PATH = "C:\\Users\\BioSystems\\Desktop\\testFolder\\workerSubmissions\\"
if not os.path.exists(SUBMISSIONS_PATH):
os.makedirs(SUBMISSIONS_PATH)
......@@ -75,7 +76,7 @@ def start():
logger.info("saving file " + destination)
file.save(destination)
subprocess.Popen(["python", PROCESS_PATH, SUBMISSIONS_PATH, RESULTS_PATH])
subprocess.Popen(["python", PROCESS_PATH, SUBMISSIONS_PATH, RESULTS_PATH, "0"])
running = True
......@@ -128,6 +129,30 @@ def retrieveLogs():
return send_file(logsZip, as_attachment=True, attachment_filename='logs.zip')
@app.route('/update_data')
def update_data():
logger.info('Request from manager to uptade internal data!')
global CURRENT_MD5
md5 = os.system("curl -s " + DATA_BASE_URL + "checksum.md5")
if CURRENT_MD5 is None or CURRENT_MD5 != md5:
logger.info('Different hash found! Updating...')
CURRENT_MD5 = md5
subprocess.Popen(["python", PROCESS_PATH, SUBMISSIONS_PATH, RESULTS_PATH, "1"])
return jsonify({"result": "updating..."}), 202
else:
msg = 'Same hash found! Aborting uptade...'
logger.info(msg)
return jsonify({"result": msg}), 202
if __name__ == "__main__":
app.run(host="0.0.0.0", port=80, threaded=False, debug=False)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment