Git Lab CI for docker build enabled! You can enable it using .gitlab-ci.yml in your project. Check file template at https://gitlab.bio.di.uminho.pt/snippets/5

Commit bd8f7a87 authored by Emanuel Cunha's avatar Emanuel Cunha
Browse files

bigg

parent 0652dcb2
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
org.eclipse.jdt.core.compiler.compliance=1.8
org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
org.eclipse.jdt.core.compiler.release=disabled
org.eclipse.jdt.core.compiler.source=1.8
......@@ -30,7 +30,7 @@ public class BiggMain {
if(args.length>0 && args[0].equals("0")) {
System.out.println("Veryfying checksum for worker initialization");
verifyChecksum();
// verifyChecksum();
}else{
performBlast();
}
......@@ -40,7 +40,7 @@ public class BiggMain {
private static void performBlast() throws NoSuchAlgorithmException, IOException {
System.out.println("Veryfying checksum for Blast execution");
verifyChecksum();
// verifyChecksum();
System.out.println("Unzipping necessary files");
unzipFiles();
......@@ -132,10 +132,10 @@ public class BiggMain {
if (!checksumFile.exists()) {
System.out.println("Checksum not found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
URL urlMd5 = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
URL urlBiggFiles = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
}
......@@ -145,16 +145,14 @@ public class BiggMain {
if(!verify) {
System.out.println("Different checksum found - downloading checksum + files");
URL urlMd5 = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
URL urlMd5 = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
org.apache.commons.io.FileUtils.copyURLToFile(urlMd5, checksumFile);
File biggFilesFile = new File(biggFilesZip);
URL urlBiggFiles = new URL("https://merlin-sysbio.org/data/bigg/genericDump/BiggDatabaseDump.zip");
URL urlBiggFiles = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/BiggDatabaseDump.zip");
org.apache.commons.io.FileUtils.copyURLToFile(urlBiggFiles, biggFilesFile);
}else
System.out.println("Checksums are equal - no need to download anything from FTP");
}
}
......@@ -167,7 +165,7 @@ public class BiggMain {
*/
private static boolean verifyKeys() throws IOException, NoSuchAlgorithmException {
URL url = new URL("https://merlin-sysbio.org/data/bigg/genericDump/checksum.md5");
URL url = new URL("https://merlin.bio.di.uminho.pt/data/bigg/genericDump/checksum.md5");
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
BufferedReader rd = new BufferedReader(new InputStreamReader(conn.getInputStream()));
......
......@@ -43,7 +43,7 @@ public class UpdateBiggFiles {
private static boolean resultsNeedUpdate = false;
private static List<String> modelBiggIds = new ArrayList<String>();
//Docker Paths
// private static String generalPath = "C:/Users/Asus/Desktop/";
// private static String generalPath = "C:/Users/Asus/Desktop/";
private static String generalPath = BiggUtils.getRootDirectory();//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String mainPath = generalPath + "Bigg_Files/";//BiggUtils.getRootDirectory(); //Main path where all the files will be
private static String modelsPath = mainPath.concat("BiggData/"); //Folder where the models files will be
......@@ -55,60 +55,60 @@ public class UpdateBiggFiles {
private static File resultsFile = new File(resultsPath);
private static Map<String,String> updatedModels = new HashMap<String,String>();
private static Map<String,List<String>> reactionsAliases = new HashMap<String,List<String>>();
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
unzipBiggData();
String resultModel = BiggUtils.fetch("models");
System.out.println("----starting get models details-----");
getModelsDetails(resultModel);
//if(resultsNeedUpdate) {
writeHistoryFile();
exportBiggDatabase();
verifyResultsFolder();
System.out.println("----reactions aliases file-----");
writeReactionsAliases();
System.out.println("----genes reaction relation-----");
genesReactionsRelation();
System.out.println("----creating fasta file-----");
createFastaFile(resultModel);
System.out.println("----Seq Id reaction relation-----");
writeSeqIDReactionRelationFile();
System.out.println("----Genes reactions rule file-----");
genesReactionsRuleFile();
System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel);
zipBiggFilesFolder();
writeHistoryFile();
exportBiggDatabase();
verifyResultsFolder();
System.out.println("----reactions aliases file-----");
writeReactionsAliases();
System.out.println("----genes reaction relation-----");
genesReactionsRelation();
System.out.println("----creating fasta file-----");
createFastaFile(resultModel);
System.out.println("----Seq Id reaction relation-----");
writeSeqIDReactionRelationFile();
System.out.println("----Genes reactions rule file-----");
genesReactionsRuleFile();
System.out.println("----Models organisms file-----");
getModelsOrganismsFile(resultModel);
zipBiggFilesFolder();
//}
}
public UpdateBiggFiles(String model) {
try {
this.setModel(model);
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/**
*
* @param resultModel
......@@ -117,9 +117,9 @@ public class UpdateBiggFiles {
private static void getModelsDetails(String resultModel) throws Exception {
JSONObject jsonObject = new JSONObject(resultModel);
getModelsBiggIds(jsonObject);
boolean forceGetInfo = false;
if(!FileUtils.existsPath(modelsPath)) {
//Create BiggData folder if it doesn't exist yet
......@@ -132,33 +132,33 @@ public class UpdateBiggFiles {
forceGetInfo = true;
}
DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss");
Date date = new Date();
String backupFolder = historyPath.concat(dateFormat.format(date) + "_BACKUP/");
boolean alreadyBackedUp = false;
for (String model : modelBiggIds) {
//Get current files and check if the file already exists
if(!forceGetInfo && checkModelFileExists(model,modelsPath)) {
//Get date from file and check if last_updated date is superior
if(checkModelFileLastUpdated(model))
continue;
else
updatedModels.put(model, "updated");
}else
updatedModels.put(model, "created");
if(!alreadyBackedUp) {
createBackup(backupFolder);
alreadyBackedUp=true;
}
resultsNeedUpdate=true; //starts as false. If there is at least one model that needs update, change the flag to true, in order to indicate that the results need to be downloaded again
JSONObject fullDetailsModel = new JSONObject();
JSONObject fullDetailsModelComplete = new JSONObject();
......@@ -169,7 +169,7 @@ public class UpdateBiggFiles {
fullDetailsModel.put("organism", "");
else
fullDetailsModel.put("organism", modelDetails.getString("organism"));
if(modelDetails.get("last_updated").equals(null))
fullDetailsModel.put("last_updated", "");
else
......@@ -299,7 +299,7 @@ public class UpdateBiggFiles {
fullDetailsModel.put("reaction", fullDetailsReaction);
fullDetailsModelComplete.put(model, fullDetailsModel);
//Write model file
String jsonToFile = fullDetailsModelComplete.toString();
PrintWriter out = new PrintWriter(modelsPath+model+".txt");
......@@ -309,7 +309,7 @@ public class UpdateBiggFiles {
}
}
/**
*
* @param model
......@@ -318,12 +318,12 @@ public class UpdateBiggFiles {
* @throws IOException
*/
private static boolean checkModelFileExists(String model, String unzippedPath) throws IOException {
File unzippedFile;
unzippedFile = new File(unzippedPath);
for(File file : unzippedFile.listFiles()) {
if(file.getName().equals(model+".txt")) {
......@@ -331,11 +331,11 @@ public class UpdateBiggFiles {
}
}
return false;
}
/**
*
* @param model
......@@ -343,26 +343,26 @@ public class UpdateBiggFiles {
* @throws Exception
*/
private static boolean checkModelFileLastUpdated(String model) throws Exception {
boolean fileExists = false;
String modelDetailsStr = BiggUtils.fetch("models", model);
JSONObject modelDetails = new JSONObject(modelDetailsStr);
String stringDateWeb = modelDetails.get("last_updated").toString();
BiggModels modelInfo = readSpecificModelDetails(model);
String stringDateLocal = modelInfo.getLastUpdated();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMM d, yyyy", Locale.ENGLISH);
LocalDate dateWeb = LocalDate.parse(stringDateWeb, formatter);
LocalDate dateLocal = LocalDate.parse(stringDateLocal,formatter);
if(!dateWeb.isAfter(dateLocal))
fileExists = true;
return fileExists;
}
......@@ -376,52 +376,52 @@ public class UpdateBiggFiles {
public static BiggModels readSpecificModelDetails(String model) throws Exception{
BiggModels biggModel = new BiggModels();
try {
File f = new File(modelsPath+model+".txt");
if(!f.exists())
throw new NoSuchFileException("File not found");
String file = BiggUtils.readFile(mainPath+"BiggData/"+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
modelDetails = modelDetails.getJSONObject(model);
try {
File f = new File(modelsPath+model+".txt");
if(!f.exists())
throw new NoSuchFileException("File not found");
//model details
String file = BiggUtils.readFile(mainPath+"BiggData/"+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
modelDetails = modelDetails.getJSONObject(model);
biggModel.setBiggId(model);
//comentado para testar o load bigg data. Descomentar depois.
biggModel.setLastUpdated(BiggUtils.validateString("last_updated", modelDetails));
biggModel.setOrganism(BiggUtils.validateString("organism", modelDetails));
biggModel.setReferenceId( BiggUtils.validateString("reference_id", modelDetails));
biggModel.setReferenceType(BiggUtils.validateString("reference_type", modelDetails));
biggModel.setGenomeRefString(BiggUtils.validateString("genome_ref_string", modelDetails));
biggModel.setGenomeName(BiggUtils.validateString("genome_name", modelDetails));
//model details
biggModel.setEscher_maps(BiggUtils.getEscherMaps(modelDetails));
//model metabolites
biggModel.setMetabolites(readModelMetabolites(modelDetails.getJSONObject("metabolite"), model));
biggModel.setBiggId(model);
//model reactions
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction")));
//comentado para testar o load bigg data. Descomentar depois.
biggModel.setLastUpdated(BiggUtils.validateString("last_updated", modelDetails));
biggModel.setOrganism(BiggUtils.validateString("organism", modelDetails));
biggModel.setReferenceId( BiggUtils.validateString("reference_id", modelDetails));
biggModel.setReferenceType(BiggUtils.validateString("reference_type", modelDetails));
biggModel.setGenomeRefString(BiggUtils.validateString("genome_ref_string", modelDetails));
biggModel.setGenomeName(BiggUtils.validateString("genome_name", modelDetails));
//model genes
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
biggModel.setEscher_maps(BiggUtils.getEscherMaps(modelDetails));
//model metabolites
biggModel.setMetabolites(readModelMetabolites(modelDetails.getJSONObject("metabolite"), model));
//model reactions
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction")));
//model genes
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
}catch(IOException e) {
System.out.println("File not found");
}catch(Exception ex) {
System.out.println(ex.getMessage());
}
}catch(IOException e) {
System.out.println("File not found");
}catch(Exception ex) {
System.out.println(ex.getMessage());
}
System.out.println("END");
return biggModel;
}
/**
......@@ -438,7 +438,7 @@ public class UpdateBiggFiles {
File f = new File(modelsPath+model+".txt");
if(!f.exists())
continue;
String file = BiggUtils.readFile(modelsPath+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
......@@ -448,30 +448,30 @@ public class UpdateBiggFiles {
//model genes
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
biggModel.setReactions(readModelReactions(modelDetails.getJSONObject("reaction")));
List<BiggReactions> biggReactions = biggModel.getReactions();
for(BiggGenes gene : biggModel.getGenes()) {
List<String> listReactionsBiggIds = new ArrayList<String>();
for(BiggReactions reaction : gene.getReaction()) {
for(BiggReactions reac : biggReactions) {
if(reac.getBiggId().equals(reaction.getBiggId())) {
for(BiggReactionResult result : reac.getResults()) {
listReactionsBiggIds.add(result.getExportedReactionId());
}
}
}
}
if(geneReactions.has(gene.getBiggId())) {
JSONArray jsonReactions = (JSONArray) geneReactions.get(gene.getBiggId());
ArrayList<String> currentReactions = new ArrayList<String>();
JSONArray jArray = (JSONArray)jsonReactions;
......@@ -485,17 +485,17 @@ public class UpdateBiggFiles {
if(!currentReactions.contains(reaction))
currentReactions.add(reaction);
}
geneReactions.remove(gene.getBiggId());
geneReactions.put(gene.getBiggId(), currentReactions);
}else
geneReactions.put(gene.getBiggId(), listReactionsBiggIds);
}
}
......@@ -504,7 +504,7 @@ public class UpdateBiggFiles {
PrintWriter out = new PrintWriter(resultsPath+"geneReactionRelation.txt");
out.print(jsonToFile);
out.close();
} catch (IOException e) {
......@@ -512,14 +512,14 @@ public class UpdateBiggFiles {
e.printStackTrace();
}
}
/**
*
* @param resultModel
* @throws Exception
*/
private static void createFastaFile(String resultModel) throws Exception{
List<String> addedGenes = new ArrayList<String>();
List<String> addedSequences = new ArrayList<String>();
List<BiggGenes> repeatedSequences = new ArrayList<BiggGenes>();
......@@ -530,19 +530,19 @@ public class UpdateBiggFiles {
PrintWriter writerSequences = new PrintWriter(resultsPath+"sequenceIdsGenesRelation.txt");
String currentSeqId="";
int idNr=1;
for (String model : modelBiggIds) {
try {
//get current model details
File f = new File(modelsPath+model+".txt");
if(!f.exists())
continue;
String file = BiggUtils.readFile(modelsPath+model+".txt", StandardCharsets.UTF_8);
System.out.println(model);
JSONObject modelDetails = new JSONObject(file);
modelDetails = modelDetails.getJSONObject(model);
BiggModels biggModel = new BiggModels();
......@@ -551,15 +551,15 @@ public class UpdateBiggFiles {
biggModel.setGenes(readModelGenes(modelDetails.getJSONObject("gene")));
for(BiggGenes gene:biggModel.getGenes()){
//to check if this gene was already validated in another model
if(addedGenes.contains(gene.getBiggId()))
continue;
else
addedGenes.add(gene.getBiggId());
String sequence = "";
if(gene.getProteinSequence() != null && gene.getProteinSequence() != ""){
sequence = gene.getProteinSequence();
}else {
......@@ -575,7 +575,7 @@ public class UpdateBiggFiles {
symL = RNATools.translate(symL);
sequence = symL.seqString();
sequence = sequence.replace("*", "");
}catch (IllegalAlphabetException ex) {
System.out.println(ex.getMessage());
......@@ -586,7 +586,7 @@ public class UpdateBiggFiles {
}
}
}
if(!addedSequences.contains(sequence)){
//first time encounter
addedSequences.add(sequence);
......@@ -594,13 +594,13 @@ public class UpdateBiggFiles {
currentSeqId = formatted;
sequenceIdsProteinSequence.put(sequence,formatted );
sequenceIds.put(formatted, gene.getBiggId());
writeFastaFile(writer, sequence, currentSeqId);
idNr++;
}else{
//already exists - writes in another file (repeatedSequences)
String seqId = sequenceIdsProteinSequence.get(sequence);
if(!repeatedSequences.contains(gene)) {
......@@ -609,24 +609,24 @@ public class UpdateBiggFiles {
String newGeneVal = sequenceIds.get(seqId) +"; "+ gene.getBiggId();
sequenceIds.put(seqId,newGeneVal);
}
}
}
}