Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit 0652dcb2 authored by Cláudia Isabela Sampaio Ganâncio's avatar Cláudia Isabela Sampaio Ganâncio
Browse files

files added to FTP server

parent 625116e1
#FileLock
#Fri Jul 24 18:26:10 BST 2020
hostName=172.20.67.7
id=17381dc5ce781906b635d22dc33f9fbb9104380bda8
method=file
server=172.20.67.7\:56230
2020-07-21 19:29:16 jdbc[35]: exception
org.h2.jdbc.JdbcSQLSyntaxErrorException: Schema "UPDATEDBIGGDATABASE" not found; SQL statement:
DROP SCHEMA updatedbiggdatabase CASCADE; [90079-200]
at org.h2.message.DbException.getJdbcSQLException(DbException.java:576)
at org.h2.message.DbException.getJdbcSQLException(DbException.java:429)
at org.h2.message.DbException.get(DbException.java:205)
at org.h2.message.DbException.get(DbException.java:181)
at org.h2.command.ddl.DropSchema.update(DropSchema.java:46)
at org.h2.command.CommandContainer.update(CommandContainer.java:198)
at org.h2.command.Command.executeUpdate(Command.java:251)
at org.h2.jdbc.JdbcStatement.executeInternal(JdbcStatement.java:228)
at org.h2.jdbc.JdbcStatement.execute(JdbcStatement.java:201)
at pt.uminho.ceb.biosystems.merlin.dataAccess.H2DatabaseAccess.dropDatabase(H2DatabaseAccess.java:243)
at pt.uminho.ceb.biosystems.merlin.dataAccess.InitDataAccess.dropDatabase(InitDataAccess.java:170)
at pt.uminho.ceb.biosystems.merlin.services.DatabaseServices.dropDatabase(DatabaseServices.java:107)
at pt.uminho.ceb.biosystems.bigg.metabolic.loader.blast.ProvideBiggReactionsToGenes.loadDatabase(ProvideBiggReactionsToGenes.java:594)
at pt.uminho.ceb.biosystems.bigg.metabolic.loader.blast.ProvideBiggReactionsToGenes.<init>(ProvideBiggReactionsToGenes.java:80)
at pt.uminho.ceb.biosystems.bigg.metabolic.loader.blast.BiggMain.main(BiggMain.java:70)
......@@ -4,6 +4,10 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
......@@ -18,10 +22,29 @@ import pt.uminho.ceb.biosystems.merlin.utilities.io.FileUtils;
public class BiggMain {
private static String workFolderID = "/workdir/";
// private static String workFolderID = "C:/Users/Asus/Desktop/"; //Docker (/workdir/)
public static void main(String[] args) throws Exception {
String workFolderID = "/workdir/";
// String workFolderID = "C:/Users/Asus/Desktop/"; //Docker (/workdir/)
if(args.length>0 && args[0].equals("0")) {
System.out.println("Veryfying checksum for worker initialization");
verifyChecksum();
}else{
performBlast();
}
}
private static void performBlast() throws NoSuchAlgorithmException, IOException {
System.out.println("Veryfying checksum for Blast execution");
verifyChecksum();
System.out.println("Unzipping necessary files");
unzipFiles();
String paramsPath = workFolderID.concat("workerSubmissions/params.txt");
Logger logger = Logger.getLogger("BiggMain");
......@@ -33,8 +56,6 @@ public class BiggMain {
SimpleFormatter formatter = new SimpleFormatter();
fh.setFormatter(formatter);
FileUtils.extractZipFile(workFolderID+"Bigg_Files/Results.tgz", workFolderID+"Bigg_Files/");
List<String> listArgs = readParamsFile(paramsPath);
logger.info("Getting properties");
......@@ -68,14 +89,136 @@ public class BiggMain {
logger.info("Starting ProvideBiggReactionsToGenes");
new ProvideBiggReactionsToGenes(workFolderID, filePath, properties, option, items, includeReactionsWithoutGPRBigg, includeReactionsWithoutGPR);
}
/*
* Unzip all files to be used during the whole process. All the files will be deleted in the end, and only the zip will stay, as expected
* */
private static void unzipFiles() throws IOException {
// TODO Auto-generated method stub
String biggFilesZip = workFolderID + "Bigg_Files.zip";
String biggFilesPath = workFolderID + "Bigg_Files/";
String biggDataZip = workFolderID + "Bigg_Files/BiggData.tgz";
String biggDatabaseDumpZip = workFolderID + "Bigg_Files/BiggDatabaseDump.zip";
String biggResultsZip = workFolderID + "Bigg_Files/Results.tgz";
if(FileUtils.existsPath(biggFilesZip)) {
FileUtils.extractZipFile(biggFilesZip, workFolderID);
if(FileUtils.existsPath(biggDataZip))
FileUtils.extractZipFile(biggDataZip, biggFilesPath);
if(FileUtils.existsPath(biggResultsZip))
FileUtils.extractZipFile(biggResultsZip, biggFilesPath);
if(FileUtils.existsPath(biggDatabaseDumpZip))
FileUtils.extractZipFile(biggDatabaseDumpZip, biggFilesPath);
}
}
File unzippedResults = new File(workFolderID+"Bigg_Files/Results/");
private static void verifyChecksum() throws IOException, NoSuchAlgorithmException {
// TODO Auto-generated method stub
FileUtils.deleteDirectory(unzippedResults);
boolean verify = false;
String biggFilesZip = workFolderID + "Bigg_Files.zip";
File checksumFile = new File(workFolderID.concat("checksum.md5"));
if (!checksumFile.exists()) {
System.out.println("Checksum not found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
}
else{
verify=verifyKeys();
System.out.println("The result of the verification of md5 file was " + Boolean.toString(verify));
if(!verify) {
System.out.println("Different checksum found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
}else
System.out.println("Checksums are equal - no need to download anything from FTP");
}
}
/**
* This method verifies the key in the md5 file and checks whether the results were corrupted or not.
* @return boolean informing whether the results were corrupted or not.
* @throws IOException
* @throws NoSuchAlgorithmException
*/
private static boolean verifyKeys() throws IOException, NoSuchAlgorithmException {
URL url = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String md5KeyFTP = rd.readLine();
String md5KeyLocal = readWordInFile(workFolderID + "checksum.md5");
if(md5KeyFTP.equals(md5KeyLocal)) {
return true;
}
else {
System.out.println("The checksums are not equal.");
return false;
}
}
/**
* This method read a word in a file
* @param path: {@code String} with the path for the file
* @return String which is the word in the file
*/
public static String readWordInFile(String path){
try {
BufferedReader reader = new BufferedReader(new FileReader(path));
String line;
while ((line = reader.readLine()) != null) {
if(!line.isEmpty() && !line.contains("**")) {
reader.close();
return line.trim();
}
}
reader.close();
}
catch (IOException e) {
e.printStackTrace();
}
return null;
}
public static List<String> readParamsFile(String path) {
List<String> listArgs = new ArrayList<String>();
......
......@@ -65,6 +65,8 @@ public class ProvideBiggReactionsToGenes {
this.models = items;
try {
Blast blast = new Blast(workFolderID, queryPath, properties, option, items);
blastResults = blast.getResults();
......@@ -524,38 +526,7 @@ public class ProvideBiggReactionsToGenes {
if(h2DatabaseFile.exists())
org.apache.commons.io.FileUtils.deleteDirectory(h2DatabaseFile);
AtomicBoolean cancel = new AtomicBoolean(false);
boolean verify = false;
String biggFilesPath = workFolderID + "Bigg_Files/";
String biggDatabaseDumpZip = workFolderID + "Bigg_Files/BiggDatabaseDump.zip";
File checksumFile = new File(biggFilesPath.concat("checksum.md5"));
if (!checksumFile.exists()) {
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggDatabaseDumpFile = new File(biggDatabaseDumpZip);
URL urlBiggDatabaseDump = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggDatabaseDump, biggDatabaseDumpFile);
}
else{
verify=verifyKeys();
System.out.println("The result of the verification of md5 file was " + Boolean.toString(verify));
if(!verify) {
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggDatabaseDumpFile = new File(biggDatabaseDumpZip);
URL urlBiggDatabaseDump = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggDatabaseDump, biggDatabaseDumpFile);
}
}
//Extract zip
FileUtils.extractZipFile(biggDatabaseDumpZip, biggFilesPath);
System.out.println("-----READING XML DB------");
DatabaseServices.generateDatabase(workspaceName);
......@@ -743,63 +714,4 @@ public class ProvideBiggReactionsToGenes {
}
/**
* This method verifies the key in the md5 file and checks whether the results were corrupted or not.
* @return boolean informing whether the results were corrupted or not.
* @throws IOException
* @throws NoSuchAlgorithmException
*/
private boolean verifyKeys() throws IOException, NoSuchAlgorithmException {
URL url = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String md5KeyFTP = rd.readLine();
String md5KeyLocal = readWordInFile(workFolderID + "Bigg_Files/checksum.md5");
if(md5KeyFTP.equals(md5KeyLocal)) {
return true;
}
else {
System.out.println("The checksums are not equal.");
return false;
}
}
/**
* This method read a word in a file
* @param path: {@code String} with the path for the file
* @return String which is the word in the file
*/
public static String readWordInFile(String path){
try {
BufferedReader reader = new BufferedReader(new FileReader(path));
String line;
while ((line = reader.readLine()) != null) {
if(!line.isEmpty() && !line.contains("**")) {
reader.close();
return line.trim();
}
}
reader.close();
}
catch (IOException e) {
e.printStackTrace();
}
return null;
}
}
......@@ -43,12 +43,14 @@ public class UpdateBiggFiles {
private static boolean resultsNeedUpdate = false;
private static List<String> modelBiggIds = new ArrayList<String>();
//Docker Paths
private static String mainPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be
// private static String mainPath = "C:/Users/Asus/Desktop/Bigg_Files/";//BiggUtils.getRootDirectory(); //Main path where all the files will be
// private static String generalPath = "C:/Users/Asus/Desktop/";
private static String generalPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String mainPath = generalPath + "Bigg_Files/";//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String modelsPath = mainPath.concat("BiggData/"); //Folder where the models files will be
private static String resultsPath = mainPath.concat("Results/"); //Folder where the results files will be
private static String historyPath = mainPath.concat("History/"); //Folder where the History files will be
private static String resultsZipPath = mainPath.concat("Results.tgz"); //Results zip path
private static String historyPath = generalPath.concat("History/"); //Folder where the History files will be
private static String biggFilesZipPath = generalPath.concat("Bigg_Files.zip"); //BiggFiles zip path
private static String biggDataZipPath = mainPath.concat("BiggData.tgz"); //Results zip path
private static File modelsFile = new File(modelsPath);
private static File resultsFile = new File(resultsPath);
private static Map<String,String> updatedModels = new HashMap<String,String>();
......@@ -58,12 +60,14 @@ public class UpdateBiggFiles {
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
unzipBiggData();
String resultModel = BiggUtils.fetch("models");
System.out.println("----starting get models details-----");
getModelsDetails(resultModel);
if(resultsNeedUpdate) {
//if(resultsNeedUpdate) {
writeHistoryFile();
......@@ -84,10 +88,10 @@ public class UpdateBiggFiles {
System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel);
zipResultsFolder();
zipBiggFilesFolder();
}
//}
......@@ -1101,13 +1105,28 @@ public class UpdateBiggFiles {
}
private static void zipResultsFolder() throws IOException {
private static void zipBiggFilesFolder() throws IOException {
if(FileUtils.existsPath(resultsPath)) {
FileUtils.createZipFile(resultsPath, resultsZipPath, 1);
FileUtils.deleteDirectory(resultsFile);
if(FileUtils.existsPath(modelsPath)) {
FileUtils.createZipFile(modelsPath, biggDataZipPath, 1);
File modelsPathFile = new File(modelsPath);
BiggUtils.deleteDirectory(modelsPathFile);
}
if(FileUtils.existsPath(mainPath)) {
FileUtils.createZipFile(mainPath, biggFilesZipPath, 1);
}
}
private static void unzipBiggData() throws IOException {
if(FileUtils.existsPath(biggDataZipPath)) {
FileUtils.extractZipFile(biggDataZipPath, mainPath);
}
}
......@@ -1318,8 +1337,7 @@ public class UpdateBiggFiles {
if(!file.exists())
file.mkdir();
org.apache.commons.io.FileUtils.copyDirectory(new File(modelsPath), new File(folderPath + "BiggData/"));
org.apache.commons.io.FileUtils.copyFile(new File(resultsZipPath), new File(folderPath + "Results.tgz"));
org.apache.commons.io.FileUtils.copyFile(new File(biggFilesZipPath), new File(folderPath + "BiggFiles.tgz"));
String zipPath = folderPath;
zipPath = zipPath.substring(0, folderPath.length() - 1);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment