Commit 982c1625 authored by Davide Lagoa's avatar Davide Lagoa
Browse files

initial commit

parents
Pipeline #260 canceled with stages
FROM python:3-onbuild
COPY . /usr/src/app
WORKDIR /workdir
EXPOSE 80
CMD ["python", "/home/workersManager.py"]
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="Flask">
<option name="enabled" value="true" />
</component>
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="Python 3.6 (untitled1)" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="TemplatesService">
<option name="TEMPLATE_CONFIGURATION" value="Jinja2" />
<option name="TEMPLATE_FOLDERS">
<list>
<option value="$MODULE_DIR$/templates" />
</list>
</option>
</component>
<component name="TestRunnerService">
<option name="PROJECT_TEST_RUNNER" value="Unittests" />
</component>
</module>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="JavaScriptSettings">
<option name="languageLevel" value="ES6" />
</component>
<component name="ProjectRootManager" version="2" project-jdk-name="Python 3.6 (untitled1)" project-jdk-type="Python SDK" />
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/TransytLatestVersion.iml" filepath="$PROJECT_DIR$/.idea/TransytLatestVersion.iml" />
</modules>
</component>
</project>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ChangeListManager">
<list default="true" id="b7761742-cf0b-462b-a78e-3cbc6fd32d3b" name="Default Changelist" comment="" />
<option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileEditorManager">
<leaf>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/manager.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="356">
<caret line="217" column="11" lean-forward="true" selection-start-line="217" selection-start-column="11" selection-end-line="217" selection-end-column="11" />
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/templates/status.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-664" />
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="true">
<entry file="file://$PROJECT_DIR$/templates/download.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="456">
<caret line="52" column="6" lean-forward="true" selection-start-line="52" selection-start-column="6" selection-end-line="52" selection-end-column="6" />
<folding>
<element signature="n#style#0;n#body#0;n#html#0;n#!!top" expanded="true" />
</folding>
</state>
</provider>
</entry>
</file>
<file pinned="false" current-in-tab="false">
<entry file="file://$PROJECT_DIR$/static/css/downloadButton.css">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="80">
<caret line="4" column="21" lean-forward="true" selection-start-line="4" selection-start-column="21" selection-end-line="4" selection-end-column="21" />
</state>
</provider>
</entry>
</file>
</leaf>
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="CSS File" />
</list>
</option>
</component>
<component name="FindInProjectRecents">
<findStrings>
<find>debug</find>
</findStrings>
</component>
<component name="IdeDocumentHistory">
<option name="CHANGED_PATHS">
<list>
<option value="$PROJECT_DIR$/manager.py" />
<option value="$PROJECT_DIR$/static/css/downloadButton.css" />
<option value="$PROJECT_DIR$/templates/download.html" />
</list>
</option>
</component>
<component name="ProjectFrameBounds" extendedState="7">
<option name="x" value="2" />
<option name="y" value="-918" />
<option name="width" value="1378" />
<option name="height" value="735" />
</component>
<component name="ProjectView">
<navigator proportions="" version="1">
<foldersAlwaysOnTop value="true" />
</navigator>
<panes>
<pane id="ProjectPane">
<subPane>
<expand>
<path>
<item name="TransytLatestVersion" type="b2602c69:ProjectViewProjectNode" />
<item name="TransytLatestVersion" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="TransytLatestVersion" type="b2602c69:ProjectViewProjectNode" />
<item name="TransytLatestVersion" type="462c0819:PsiDirectoryNode" />
<item name="static" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="TransytLatestVersion" type="b2602c69:ProjectViewProjectNode" />
<item name="TransytLatestVersion" type="462c0819:PsiDirectoryNode" />
<item name="static" type="462c0819:PsiDirectoryNode" />
<item name="css" type="462c0819:PsiDirectoryNode" />
</path>
<path>
<item name="TransytLatestVersion" type="b2602c69:ProjectViewProjectNode" />
<item name="TransytLatestVersion" type="462c0819:PsiDirectoryNode" />
<item name="templates" type="462c0819:PsiDirectoryNode" />
</path>
</expand>
<select />
</subPane>
</pane>
<pane id="Scope" />
</panes>
</component>
<component name="PropertiesComponent">
<property name="WebServerToolWindowFactoryState" value="false" />
<property name="last_opened_file_path" value="$PROJECT_DIR$/templates" />
<property name="list.type.of.created.stylesheet" value="CSS" />
<property name="nodejs_interpreter_path.stuck_in_default_project" value="undefined stuck path" />
<property name="nodejs_npm_path_reset_for_default_project" value="true" />
<property name="settings.editor.selected.configurable" value="com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable" />
</component>
<component name="RecentsManager">
<key name="CopyFile.RECENT_KEYS">
<recent name="C:\Users\Diogo\Desktop\TransytLatestVersion\templates" />
</key>
</component>
<component name="RunDashboard">
<option name="ruleStates">
<list>
<RuleState>
<option name="name" value="ConfigurationTypeDashboardGroupingRule" />
</RuleState>
<RuleState>
<option name="name" value="StatusDashboardGroupingRule" />
</RuleState>
</list>
</option>
</component>
<component name="RunManager" selected="Flask server.Flask (manager.py)">
<configuration name="Flask (manager.py)" type="Python.FlaskServer" temporary="true" nameIsGenerated="true">
<module name="TransytLatestVersion" />
<option name="target" value="$PROJECT_DIR$/manager.py" />
<option name="targetType" value="PATH" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="launchJavascriptDebuger" value="false" />
<method v="2" />
</configuration>
<configuration name="TransytLatestVersion" type="Python.FlaskServer">
<module name="TransytLatestVersion" />
<option name="target" value="$PROJECT_DIR$/manager.py" />
<option name="targetType" value="PATH" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="launchJavascriptDebuger" value="false" />
<method v="2" />
</configuration>
<recent_temporary>
<list>
<item itemvalue="Flask server.Flask (manager.py)" />
</list>
</recent_temporary>
</component>
<component name="SvnConfiguration">
<configuration />
</component>
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="b7761742-cf0b-462b-a78e-3cbc6fd32d3b" name="Default Changelist" comment="" />
<created>1570451179714</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1570451179714</updated>
<workItem from="1570451195629" duration="3284000" />
</task>
<servers />
</component>
<component name="TimeTrackingManager">
<option name="totallyTimeSpent" value="3284000" />
</component>
<component name="ToolWindowManager">
<frame x="-90" y="-908" width="1456" height="876" extended-state="6" />
<editor active="true" />
<layout>
<window_info id="Favorites" order="0" side_tool="true" />
<window_info active="true" content_ui="combo" id="Project" order="1" visible="true" weight="0.13610315" />
<window_info id="Structure" order="2" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Database Changes" order="0" />
<window_info anchor="bottom" id="Terminal" order="1" />
<window_info anchor="bottom" id="Event Log" order="2" side_tool="true" />
<window_info anchor="bottom" id="Version Control" order="3" />
<window_info anchor="bottom" id="Python Console" order="4" />
<window_info anchor="bottom" id="Docker" order="5" show_stripe_button="false" />
<window_info anchor="bottom" id="Message" order="6" />
<window_info anchor="bottom" id="Find" order="7" />
<window_info anchor="bottom" id="Run" order="8" visible="true" weight="0.32974428" />
<window_info anchor="bottom" id="Debug" order="9" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="10" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="11" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="12" />
<window_info anchor="right" id="Database" order="0" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="1" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="SciView" order="2" />
<window_info anchor="right" id="Ant Build" order="3" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="4" weight="0.25" />
</layout>
<layout-to-restore>
<window_info id="Favorites" order="0" side_tool="true" />
<window_info content_ui="combo" id="Project" order="1" visible="true" weight="0.13610315" />
<window_info id="Structure" order="2" side_tool="true" weight="0.25" />
<window_info anchor="bottom" id="Database Changes" order="0" />
<window_info anchor="bottom" id="Terminal" order="1" />
<window_info anchor="bottom" id="Event Log" order="2" side_tool="true" />
<window_info anchor="bottom" id="Version Control" order="3" />
<window_info anchor="bottom" id="Python Console" order="4" />
<window_info anchor="bottom" id="Docker" order="5" show_stripe_button="false" />
<window_info anchor="bottom" id="Message" order="6" />
<window_info anchor="bottom" id="Find" order="7" />
<window_info active="true" anchor="bottom" id="Run" order="8" visible="true" weight="0.32974428" />
<window_info anchor="bottom" id="Debug" order="9" weight="0.4" />
<window_info anchor="bottom" id="Cvs" order="10" weight="0.25" />
<window_info anchor="bottom" id="Inspection" order="11" weight="0.4" />
<window_info anchor="bottom" id="TODO" order="12" />
<window_info anchor="right" id="Database" order="0" />
<window_info anchor="right" id="Commander" internal_type="SLIDING" order="1" type="SLIDING" weight="0.4" />
<window_info anchor="right" id="SciView" order="2" />
<window_info anchor="right" id="Ant Build" order="3" weight="0.25" />
<window_info anchor="right" content_ui="combo" id="Hierarchy" order="4" weight="0.25" />
</layout-to-restore>
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="1" />
</component>
<component name="com.intellij.coverage.CoverageDataManagerImpl">
<SUITE FILE_PATH="coverage/TransytLatestVersion$Flask__manager_py_.coverage" NAME="Flask (manager.py) Coverage Results" MODIFIED="1570452950979" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="" />
</component>
<component name="editorHistoryManager">
<entry file="file://$PROJECT_DIR$/manager.py">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="356">
<caret line="217" column="11" lean-forward="true" selection-start-line="217" selection-start-column="11" selection-end-line="217" selection-end-column="11" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/templates/home.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-240" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/static/css/downloadButton.css">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="80">
<caret line="4" column="21" lean-forward="true" selection-start-line="4" selection-start-column="21" selection-end-line="4" selection-end-column="21" />
</state>
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/templates/status.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="-664" />
</provider>
</entry>
<entry file="file://$PROJECT_DIR$/templates/download.html">
<provider selected="true" editor-type-id="text-editor">
<state relative-caret-position="456">
<caret line="52" column="6" lean-forward="true" selection-start-line="52" selection-start-column="6" selection-end-line="52" selection-end-column="6" />
<folding>
<element signature="n#style#0;n#body#0;n#html#0;n#!!top" expanded="true" />
</folding>
</state>
</provider>
</entry>
</component>
</project>
\ No newline at end of file
#docker workers configuration
#1 - name of the workers
name = transyt_worker_
#2 - number of workers
limit = 3
#3 - workers port
port = 80
# -*- coding: utf-8 -*-
from flask import Flask, render_template, request, send_file, jsonify
import os
import socket
import shutil
import logging
logPath = '/usr/src/app/logs/'
if not os.path.exists(logPath):
os.makedirs(logPath)
logPath = logPath + 'manager.log'
logging.basicConfig(filename=logPath, level=logging.DEBUG, format='%(asctime)s: %(levelname)s: >>>%(message)s')
logging.info('Starting server')
# subprocess.Popen(["python", "/home/workersManager.py"])
app = Flask(__name__)
hostname = socket.gethostname()
SUBMISSIONS_LIMIT = 100
SUBMISSIONS_PATH = '/workdir/submissions_pool/'
PROCESSING_PATH = '/workdir/processing_pool/'
RESULTS_PATH = '/workdir/results_pool/'
last_submission_ID = 0
print('SERVER WORKING')
@app.route("/")
def index():
logging.info('Welcome page rendering...')
return render_template("home.html")
@app.route("/submit", methods=["POST"])
def submit():
logging.info('HTML submission')
files = request.files.getlist("file")
taxonomyId = request.form["text"]
validInput = verifyInput(taxonomyId)
if not validInput:
return jsonify({"Genome": "Input genome must be in .faa or .fasta format",
"Model": "Model must be in .xml format or a list of metabolites in .txt format (one metabolite per line)",
"Taxonomy ID": "Input must only contain digits"}), 406
logging.info("Taxonomy identifier: " + taxonomyId)
return run(files, taxonomyId, False)
@app.route("/submitMerlinPlugin/<taxonomyId>", methods=["POST"])
def submitMerlinPlugin(taxonomyId):
logging.info('Merlin plugin submission')
files = request.files.getlist("file")
logging.info("files in submission " + str(files))
validInput = verifyInput(taxonomyId)
if not validInput:
return jsonify({"Genome": "Input genome must be in .faa or .fasta format",
"Model": "Model must be in .xml format or a list of metabolites in .txt format (one metabolite per line)",
"Taxonomy ID": "Input must only contain digits"}), 406
return run(files, taxonomyId, True)
def run(files, taxonomyId, isRest):
logging.debug('New submission in progress...')
total = len(os.listdir(SUBMISSIONS_PATH))
if total > SUBMISSIONS_LIMIT:
logging.error('Server capacity overload! Returning error 503 to client!')
return jsonify({"result": "Server overload",
"message": "The server cannot handle the submission due to capacity overload. Please try again later!"}), 503
global last_submission_ID
last_submission_ID += 1
if not request.files.getlist("file")[0].filename:
return jsonify({"result": "rejected",
"message": "fewer files than needed submitted! Submission rejected!", }), 400
submission_path = SUBMISSIONS_PATH + str(last_submission_ID)
if os.path.exists(submission_path):
shutil.rmtree(submission_path, ignore_errors=True)
os.makedirs(submission_path)
logging.debug("Submission " + str(last_submission_ID) + " folder created!")
inputCheck = []
for file in files:
formats = {".faa":"genome.faa", ".fasta":"genome.faa", ".xml":"model.xml"}
fileName = file.filename
inFormats = False
for key in formats:
if fileName.endswith(key):
fileName = formats[key]
inFormats = True
inputCheck.append(formats[key])
if not inFormats and "genome.faa" in inputCheck and fileName.endswith(".txt"):
logging.info("ESTOU A ENTRAR AQUI")
fileName = "metabolites.txt"
inputCheck.append("metabolites.txt")
destination = "/".join([submission_path, fileName])
logging.debug('Saving file ' + fileName + ' at ' + destination)
file.save(destination)
validInput = ["genome.faa","model.xml"]
validInput2 = ["genome.faa","metabolites.txt"]
validInputs = [validInput, validInput2]
logging.info("taxID -> " + taxonomyId)
logging.info("inputCheck -> " + str(inputCheck))
if inputCheck in validInputs:
with open("/".join([submission_path,"taxID.txt"]), "w") as taxFile:
taxFile.write(taxonomyId) # write TaxonomyID file
else:
return jsonify({"Genome":"Input genome must be in .faa or .fasta format", "Model":"Model must be in .xml format or a list of metabolites in .txt format (one metabolite per line)", "Taxonomy ID":"Input must only contain digits"}), 415
with open(submission_path + "/submissionComplete", "w") as f:
f.write("Submission of files complete: ")
if len(os.listdir(submission_path)) < 4:
shutil.rmtree(submission_path)
return jsonify({"result": "rejected",
"message": "fewer files than needed submitted! Submission rejected!", }), 400
if isRest:
return jsonify({"result": "submitted",
"message": "submission accepted. Waiting in queue with " + str(total + 1) + " submissions!",
"queue": total,
"submissionID": str(last_submission_ID)}), 201
#return render_template("status.html", subId = str(last_submission_ID))
return display_msg(str(last_submission_ID), isRest)
@app.route('/status/<submissionID>/<isRest>')
def display_msg(submissionID, isRest):
if isRest == 'False':
isRest = False
elif isRest == 'True':
isRest = True
logging.info("isRest request: " + str(isRest) + " > type: " + str(type(isRest)))
logging.debug("Request to check submission " + submissionID + " status")
folders = os.listdir(SUBMISSIONS_PATH)
logging.info("submissions " + str(folders))
if submissionID in folders:
total = len(folders)
inQueue = countEntriesBelow(folders, submissionID)
if isRest:
return jsonify({"result": "queued",
"message": "Waiting in queue with " + str(total + 1) + " submissions!",
"queue": str(inQueue)}), 201
subStatus = {"result": "queued",
"message": "Waiting in queue",
"queue": "There are " + str(inQueue) + " submissions ahead of yours"}
return render_template("status.html", subId=submissionID, status=subStatus)
folders = os.listdir(PROCESSING_PATH)
logging.info("processing " + str(folders))
if submissionID in folders: # up and running response
logging.debug("Submission " + submissionID + "still running")
if isRest:
return jsonify({"result": "processing", "message": "accepted but process still running, please wait."}), 202
subStatus = {"result": "processing", "message": "accepted but process still running, please wait."}
return render_template("status.html", subId=submissionID, status=0), 202
folders = os.listdir(RESULTS_PATH)
logging.info("results " + str(folders))
if submissionID in folders:
logging.debug("Submission " + submissionID + " results complete")
if isRest:
return jsonify(
{"result": "success", "message": "the process is now complete. Download the results at ./downloads"}), 200
#return download(submissionID)
return render_template("download.html", subId = submissionID), 200
return jsonify({"result": "Server error",
"message": "Something went wrong while processing the request, please try again", }), 500
@app.route("/download/<submissionID>")
def download(submissionID):
logging.debug("Request to download submission " + submissionID + " results")
return send_file(RESULTS_PATH + submissionID + "/results.zip", as_attachment=True,
attachment_filename='results.zip')
def countEntriesBelow(l, id):
id = int(id)
res = 0
for d in l:
if int(d) == id:
return res
elif int(d) < id:
res += 1
return res
def verifyInput(userInput):
userInput = str(userInput)
validTaxID = False
if userInput:
validTaxID = True
digits = "0123456789"
for character in userInput:
if character not in digits:
validTaxID = False
return validTaxID
logging.info('Server running!')
if __name__ == "__main__":
app.run(host="0.0.0.0", port=80, threaded=True, debug=True)
.center {
margin: auto;
width: 60%;
border: 3px solid #000066;
padding: 10px;
word-wrap: break-word;
font-family: 'Oxygen';