Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit bd8f7a87 authored by Emanuel Cunha's avatar Emanuel Cunha
Browse files

bigg

parent 0652dcb2
eclipse.preferences.version=1 eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8 org.eclipse.jdt.core.compiler.source=1.8
...@@ -30,7 +30,7 @@ public class BiggMain { ...@@ -30,7 +30,7 @@ public class BiggMain {
if(args.length>0 && args[0].equals("0")) { if(args.length>0 && args[0].equals("0")) {
System.out.println("Veryfying checksum for worker initialization"); System.out.println("Veryfying checksum for worker initialization");
verifyChecksum(); // verifyChecksum();
}else{ }else{
performBlast(); performBlast();
} }
...@@ -40,7 +40,7 @@ public class BiggMain { ...@@ -40,7 +40,7 @@ public class BiggMain {
private static void performBlast() throws NoSuchAlgorithmException, IOException { private static void performBlast() throws NoSuchAlgorithmException, IOException {
System.out.println("Veryfying checksum for Blast execution"); System.out.println("Veryfying checksum for Blast execution");
verifyChecksum(); // verifyChecksum();
System.out.println("Unzipping necessary files"); System.out.println("Unzipping necessary files");
unzipFiles(); unzipFiles();
...@@ -132,10 +132,10 @@ public class BiggMain { ...@@ -132,10 +132,10 @@ public class BiggMain {
if (!checksumFile.exists()) { if (!checksumFile.exists()) {
System.out.println("Checksum not found - downloading checksum + files"); System.out.println("Checksum not found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5"); URL urlMd5 = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile); org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip); File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip"); URL urlBiggFiles = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile); org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
} }
...@@ -145,16 +145,14 @@ public class BiggMain { ...@@ -145,16 +145,14 @@ public class BiggMain {
if(!verify) { if(!verify) {
System.out.println("Different checksum found - downloading checksum + files"); System.out.println("Different checksum found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5"); URL urlMd5 = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile); org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip); File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip"); URL urlBiggFiles = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile); org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
}else }else
System.out.println("Checksums are equal - no need to download anything from FTP"); System.out.println("Checksums are equal - no need to download anything from FTP");
} }
} }
...@@ -167,7 +165,7 @@ public class BiggMain { ...@@ -167,7 +165,7 @@ public class BiggMain {
*/ */
private static boolean verifyKeys() throws IOException, NoSuchAlgorithmException { private static boolean verifyKeys() throws IOException, NoSuchAlgorithmException {
URL url = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5"); URL url = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
HttpURLConnection conn = (HttpURLConnection) url.openConnection(); HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET"); conn.setRequestMethod("GET");
BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream())); BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
......
...@@ -43,7 +43,7 @@ public class UpdateBiggFiles { ...@@ -43,7 +43,7 @@ public class UpdateBiggFiles {
private static boolean resultsNeedUpdate = false; private static boolean resultsNeedUpdate = false;
private static List<String> modelBiggIds = new ArrayList<String>(); private static List<String> modelBiggIds = new ArrayList<String>();
//Docker Paths //Docker Paths
// private static String generalPath = "C:/Users/Asus/Desktop/"; // private static String generalPath = "C:/Users/Asus/Desktop/";
private static String generalPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be private static String generalPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String mainPath = generalPath + "Bigg_Files/";//BiggUtils.getRootDirectory(); //Main path where all the files will be private static String mainPath = generalPath + "Bigg_Files/";//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String modelsPath = mainPath.concat("BiggData/"); //Folder where the models files will be private static String modelsPath = mainPath.concat("BiggData/"); //Folder where the models files will be
...@@ -55,60 +55,60 @@ public class UpdateBiggFiles { ...@@ -55,60 +55,60 @@ public class UpdateBiggFiles {
private static File resultsFile = new File(resultsPath); private static File resultsFile = new File(resultsPath);
private static Map<String,String> updatedModels = new HashMap<String,String>(); private static Map<String,String> updatedModels = new HashMap<String,String>();
private static Map<String,List<String>> reactionsAliases = new HashMap<String,List<String>>(); private static Map<String,List<String>> reactionsAliases = new HashMap<String,List<String>>();
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub // TODO Auto-generated method stub
unzipBiggData(); unzipBiggData();
String resultModel = BiggUtils.fetch("models"); String resultModel = BiggUtils.fetch("models");
System.out.println("----starting get models details-----"); System.out.println("----starting get models details-----");
getModelsDetails(resultModel); getModelsDetails(resultModel);
//if(resultsNeedUpdate) { //if(resultsNeedUpdate) {
writeHistoryFile(); writeHistoryFile();
exportBiggDatabase(); exportBiggDatabase();
verifyResultsFolder(); verifyResultsFolder();
System.out.println("----reactions aliases file-----"); System.out.println("----reactions aliases file-----");
writeReactionsAliases(); writeReactionsAliases();
System.out.println("----genes reaction relation-----"); System.out.println("----genes reaction relation-----");
genesReactionsRelation(); genesReactionsRelation();
System.out.println("----creating fasta file-----"); System.out.println("----creating fasta file-----");
createFastaFile(resultModel); createFastaFile(resultModel);
System.out.println("----Seq Id reaction relation-----"); System.out.println("----Seq Id reaction relation-----");
writeSeqIDReactionRelationFile(); writeSeqIDReactionRelationFile();
System.out.println("----Genes reactions rule file-----"); System.out.println("----Genes reactions rule file-----");
genesReactionsRuleFile(); genesReactionsRuleFile();
System.out.println("----Models organisms file-----"); System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel); getModelsOrganismsFile(resultModel);
zipBiggFilesFolder(); zipBiggFilesFolder();
//} //}
} }
public UpdateBiggFiles(String model) { public UpdateBiggFiles(String model) {
try { try {
this.setModel(model); this.setModel(model);
} catch (Exception e) { } catch (Exception e) {
// TODO Auto-generated catch block // TODO Auto-generated catch block
e.printStackTrace(); e.printStackTrace();
} }
} }
/** /**
* *
* @param resultModel * @param resultModel
...@@ -117,9 +117,9 @@ public class UpdateBiggFiles { ...@@ -117,9 +117,9 @@ public class UpdateBiggFiles {
private static void getModelsDetails(String resultModel) throws Exception { private static void getModelsDetails(String resultModel) throws Exception {
JSONObject jsonObject = new JSONObject(resultModel); JSONObject jsonObject = new JSONObject(resultModel);
getModelsBiggIds(jsonObject); getModelsBiggIds(jsonObject);
boolean forceGetInfo = false; boolean forceGetInfo = false;
if(!FileUtils.existsPath(modelsPath)) { if(!FileUtils.existsPath(modelsPath)) {
//Create BiggData folder if it doesn't exist yet //Create BiggData folder if it doesn't exist yet
...@@ -132,33 +132,33 @@ public class UpdateBiggFiles { ...@@ -132,33 +132,33 @@ public class UpdateBiggFiles {
forceGetInfo = true; forceGetInfo = true;
} }
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss"); DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
Date date = new Date(); Date date = new Date();
String backupFolder = historyPath.concat(dateFormat.format(date) + "_BACKUP/"); String backupFolder = historyPath.concat(dateFormat.format(date) + "_BACKUP/");
boolean alreadyBackedUp = false; boolean alreadyBackedUp = false;
for (String model : modelBiggIds) { for (String model : modelBiggIds) {
//Get current files and check if the file already exists //Get current files and check if the file already exists
if(!forceGetInfo && checkModelFileExists(model,modelsPath)) { if(!forceGetInfo && checkModelFileExists(model,modelsPath)) {
//Get date from file and check if last_updated date is superior //Get date from file and check if last_updated date is superior
if(checkModelFileLastUpdated(model)) if(checkModelFileLastUpdated(model))
continue; continue;
else else
updatedModels.put(model, "updated"); updatedModels.put(model, "updated");
}else }else
updatedModels.put(model, "created"); updatedModels.put(model, "created");
if(!alreadyBackedUp) { if(!alreadyBackedUp) {
createBackup(backupFolder); createBackup(backupFolder);
alreadyBackedUp=true; alreadyBackedUp=true;
} }
resultsNeedUpdate=true; //starts as false. If there is at least one model that needs update, change the flag to true, in order to indicate that the results need to be downloaded again resultsNeedUpdate=true; //starts as false. If there is at least one model that needs update, change the flag to true, in order to indicate that the results need to be downloaded again
JSONObject fullDetailsModel = new JSONObject(); JSONObject fullDetailsModel = new JSONObject();
JSONObject fullDetailsModelComplete = new JSONObject(); JSONObject fullDetailsModelComplete = new JSONObject();
...@@ -169,7 +169,7 @@ public class UpdateBiggFiles { ...@@ -169,7 +169,7 @@ public class UpdateBiggFiles {
fullDetailsModel.put("organism", ""); fullDetailsModel.put("organism", "");
else else
fullDetailsModel.put("organism", modelDetails.getString("organism")); fullDetailsModel.put("organism", modelDetails.getString("organism"));
if(modelDetails.get("last_updated").equals(null)) if(modelDetails.get("last_updated").equals(null))
fullDetailsModel.put("last_updated", ""); fullDetailsModel.put("last_updated", "");
else else
...@@ -299,7 +299,7 @@ public class UpdateBiggFiles { ...@@ -299,7 +299,7 @@ public class UpdateBiggFiles {
fullDetailsModel.put("reaction", fullDetailsReaction); fullDetailsModel.put("reaction", fullDetailsReaction);
fullDetailsModelComplete.put(model, fullDetailsModel); fullDetailsModelComplete.put(model, fullDetailsModel);
//Write model file //Write model file
String jsonToFile = fullDetailsModelComplete.toString(); String jsonToFile = fullDetailsModelComplete.toString();
PrintWriter out = new PrintWriter(modelsPath+model+".txt"); PrintWriter out = new PrintWriter(modelsPath+model+".txt");
...@@ -309,7 +309,7 @@ public class UpdateBiggFiles { ...@@ -309,7 +309,7 @@ public class UpdateBiggFiles {
} }
} }
/** /**
* *
* @param model * @param model
...@@ -318,12 +318,12 @@ public class UpdateBiggFiles { ...@@ -318,12 +318,12 @@ public class UpdateBiggFiles {
* @throws IOException * @throws IOException
*/ */
private static boolean checkModelFileExists(String model, String unzippedPath) throws IOException { private static boolean checkModelFileExists(String model, String unzippedPath) throws IOException {
File unzippedFile; File unzippedFile;
unzippedFile = new File(unzippedPath); unzippedFile = new File(unzippedPath);
for(File file : unzippedFile.listFiles()) { for(File file : unzippedFile.listFiles()) {
if(file.getName().equals(model+".txt")) { if(file.getName().equals(model+".txt")) {
...@@ -331,11 +331,11 @@ public class UpdateBiggFiles { ...@@ -331,11 +331,11 @@ public class UpdateBiggFiles {
} }
} }
return false; return false;
} }
/** /**
* *
* @param model * @param model
...@@ -343,26 +343,26 @@ public class UpdateBiggFiles { ...@@ -343,26 +343,26 @@ public class UpdateBiggFiles {
* @throws Exception * @throws Exception
*/ */
private static boolean checkModelFileLastUpdated(String model) throws Exception { private static boolean checkModelFileLastUpdated(String model) throws Exception {
boolean fileExists = false; boolean fileExists = false;
String modelDetailsStr = BiggUtils.fetch("models", model); String modelDetailsStr = BiggUtils.fetch("models", model);
JSONObject modelDetails = new JSONObject(modelDetailsStr); JSONObject modelDetails = new JSONObject(modelDetailsStr);
String stringDateWeb = modelDetails.get("last_updated").toString(); String stringDateWeb = modelDetails.get("last_updated").toString();
BiggModels modelInfo = readSpecificModelDetails(model); BiggModels modelInfo = readSpecificModelDetails(model);
String stringDateLocal = modelInfo.getLastUpdated(); String stringDateLocal = modelInfo.getLastUpdated();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM d, yyyy", Locale.ENGLISH); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM d, yyyy", Locale.ENGLISH);
LocalDate dateWeb = LocalDate.parse(stringDateWeb, formatter); LocalDate dateWeb = LocalDate.parse(stringDateWeb, formatter);
LocalDate dateLocal = LocalDate.parse(stringDateLocal,formatter); LocalDate dateLocal = LocalDate.parse(stringDateLocal,formatter);
if(!dateWeb.isAfter(dateLocal)) if(!dateWeb.isAfter(dateLocal))
fileExists = true; fileExists = true;
return fileExists; return fileExists;
} }
...@@ -376,52 +376,52 @@ public class UpdateBiggFiles { ...@@ -376,52 +376,52 @@ public class UpdateBiggFiles {
public static BiggModels readSpecificModelDetails(String model) throws Exception{ public static BiggModels readSpecificModelDetails(String model) throws Exception{
BiggModels biggModel = new BiggModels(); BiggModels biggModel = new BiggModels();
try { try {
File f = new File(modelsPath+model+".txt"); File f = new File(modelsPath+model+".txt");
if(!f.exists()) if(!f.exists())
throw new NoSuchFileException("File not found"); throw new NoSuchFileException("File not found");
String file = BiggUtils.readFile(mainPath+"BiggData/"+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
modelDetails = modelDetails.getJSONObject(model);
//model details String file = BiggUtils.readFile(mainPath+"BiggData/"+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
modelDetails = modelDetails.getJSONObject(model);
biggModel.setBiggId(model);
//comentado para testar o load bigg data. Descomentar depois. //model details
biggModel.setLastUpdated(BiggUtils.validateString("last_updated", modelDetails));
biggModel.setOrganism(BiggUtils.validateString("organism", modelDetails));
biggModel.setReferenceId( BiggUtils.validateString("reference_id", modelDetails));
biggModel.setReferenceType(BiggUtils.validateString("reference_type", modelDetails));
biggModel.setGenomeRefString(BiggUtils.validateString("genome_ref_string", modelDetails));
biggModel.setGenomeName(BiggUtils.validateString("genome_name", modelDetails));
biggModel.setEscher_maps(BiggUtils.getEscherMaps(modelDetails)); biggModel.setBiggId(model);
//model metabolites
biggModel.setMetabolites(readModelMetabolites(modelDetails.getJSONObject("metabolite"), model));
//model reactions //comentado para testar o load bigg data. Descomentar depois.
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction"))); biggModel.setLastUpdated(BiggUtils.validateString("last_updated", modelDetails));
biggModel.setOrganism(BiggUtils.validateString("organism", modelDetails));
biggModel.setReferenceId( BiggUtils.validateString("reference_id", modelDetails));
biggModel.setReferenceType(BiggUtils.validateString("reference_type", modelDetails));
biggModel.setGenomeRefString(BiggUtils.validateString("genome_ref_string", modelDetails));
biggModel.setGenomeName(BiggUtils.validateString("genome_name", modelDetails));
//model genes biggModel.setEscher_maps(BiggUtils.getEscherMaps(modelDetails));
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
//model metabolites
biggModel.setMetabolites(readModelMetabolites(modelDetails.getJSONObject("metabolite"), model));
//model reactions
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction")));
//model genes
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
}catch(IOException e) {
System.out.println("File not found"); }catch(IOException e) {
System.out.println("File not found");
}catch(Exception ex) {
System.out.println(ex.getMessage()); }catch(Exception ex) {
} System.out.println(ex.getMessage());
}
System.out.println("END"); System.out.println("END");
return biggModel; return biggModel;
} }
/** /**
...@@ -438,7 +438,7 @@ public class UpdateBiggFiles { ...@@ -438,7 +438,7 @@ public class UpdateBiggFiles {
File f = new File(modelsPath+model+".txt"); File f = new File(modelsPath+model+".txt");
if(!f.exists()) if(!f.exists())
continue; continue;
String file = BiggUtils.readFile(modelsPath+model+".txt", StandardCharsets.UTF_8); String file = BiggUtils.readFile(modelsPath+model+".txt", StandardCharsets.UTF_8);
System.out.println(model); System.out.println(model);
JSONObject modelDetails = new JSONObject(file); JSONObject modelDetails = new JSONObject(file);
...@@ -448,30 +448,30 @@ public class UpdateBiggFiles { ...@@ -448,30 +448,30 @@ public class UpdateBiggFiles {
//model genes //model genes
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene"))); biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction"))); biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction")));
List<BiggReactions> biggReactions = biggModel.getReactions(); List<BiggReactions> biggReactions = biggModel.getReactions();
for(BiggGenes gene : biggModel.getGenes()) { for(BiggGenes gene : biggModel.getGenes()) {
List<String> listReactionsBiggIds = new ArrayList<String>(); List<String> listReactionsBiggIds = new ArrayList<String>();
for(BiggReactions reaction : gene.getReaction()) { for(BiggReactions reaction : gene.getReaction()) {
for(BiggReactions reac : biggReactions) { for(BiggReactions reac : biggReactions) {
if(reac.getBiggId().equals(reaction.getBiggId())) { if(reac.getBiggId().equals(reaction.getBiggId())) {
for(BiggReactionResult result : reac.getResults()) {