List:Commits« Previous MessageNext Message »
From:Reggie Burnett Date:September 17 2012 8:04pm
Subject:bzr commit into ABSv2 branch (reggie.burnett:34)
View as plain text  
#At file:///C:/work/wex/absv2/ based on revid:iggy.galarza@stripped

   34 Reggie Burnett	2012-09-17
      small tweak to how we do generated files

    renamed:
      AbsUtility/ProductPreReq.cs => AbsUtility/ProductFile.cs
    modified:
      AbsImporter/Program.cs
      AbsUtility/AbsUtility.csproj
      AbsUtility/Product.cs
      AbsUtility/Utility.cs
      AbsUtility/ProductFile.cs
=== modified file 'AbsImporter/Program.cs'
=== modified file 'AbsImporter/Program.cs'
--- a/AbsImporter/Program.cs	2012-09-14 22:32:05 +0000
+++ b/AbsImporter/Program.cs	2012-09-17 20:04:19 +0000
@@ -34,32 +34,17 @@
     {
       string path = Path.Combine(Settings.IncomingFolder, p.Name);
       string[] files = Directory.GetFiles(path);
-      for (int i = 0; i < files.Length; i++)
-        files[i] = files[i].ToLowerInvariant();
 
       try
       {
         // no files == nothing to do
         if (files.Length == 0) return;
 
-        // perform some basic validation on the files
-        BasicFileValidation(p, files);
-
-        // now let's strip off the path names
-        List<string> filenames = new List<string>();
-        foreach (string file in files)
-          filenames.Add(Path.GetFileName(file));
-        files = filenames.ToArray();
-
-        // validate that the files are all msi and have the same version
-        if (!p.ValidateFiles(files))
-          throw new Exception("Files failed naming verification");
-
-        // now that the default files have been verified, perform any prerequisites.
-        if (!p.BuildPreReqs(Utility.GetVersion(files[0], true)))
-          throw new Exception("Failed to build all pre-reqs");
-
-        p.CreateJob(files);
+        // validate the files in the incoming folder
+        p.ValidateFiles();
+
+        // files are valid so we create the job
+        p.CreateJob();
       }
       catch (Exception ex)
       {
@@ -68,22 +53,6 @@
       }
     }
 
-    static void BasicFileValidation(Product p, string[] files)
-    {
-      // this makes sure they all end in msi, they all have the same version
-      // and they are done copying
-      if (!Utility.ValidateFiles(files))
-        throw new Exception("Either not all files are msi or are not of same version");
-
-      // not enough files, nothing to do
-      if (files.Length < p.NumJobFilesRequired)
-        throw new Exception("Not enough files for release");
-
-      // if there are too many files, we can't work so we log it and move on
-      if (files.Length > p.NumJobFilesRequired)
-        throw new Exception("Too many files for release");
-    }
-
     static bool WeAreRunning()
     {
       foreach (Process clsProcess in Process.GetProcesses())

=== modified file 'AbsUtility/AbsUtility.csproj'
--- a/AbsUtility/AbsUtility.csproj	2012-09-14 22:32:05 +0000
+++ b/AbsUtility/AbsUtility.csproj	2012-09-17 20:04:19 +0000
@@ -52,7 +52,7 @@
     <Compile Include="MSIRecord.cs" />
     <Compile Include="MSIView.cs" />
     <Compile Include="Product.cs" />
-    <Compile Include="ProductPreReq.cs" />
+    <Compile Include="ProductFile.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Properties\Resources.Designer.cs">
       <DependentUpon>Resources.resx</DependentUpon>

=== modified file 'AbsUtility/Product.cs'
--- a/AbsUtility/Product.cs	2012-09-14 22:32:05 +0000
+++ b/AbsUtility/Product.cs	2012-09-17 20:04:19 +0000
@@ -23,40 +23,17 @@
       IncomingFolder = Path.Combine(Settings.IncomingFolder, Name);
       BuildFolder = Path.Combine(Settings.BuildsFolder, Name);
 
-      Prerequisites = new List<ProductPreReq>();
-
-      x86GPLPattern = GetRegex(row["Regex_X86_GPL"]);
-      x86COMPattern = GetRegex(row["Regex_X86_COM"]);
-      x64GPLPattern = GetRegex(row["Regex_X64_GPL"]);
-      x64COMPattern = GetRegex(row["Regex_X64_COM"]);
-
-
-      string selectFormat = "SELECT * FROM (SELECT series FROM abs_products WHERE id = {0}) AS a, (SELECT * FROM abs_product_files WHERE product_id = {0}) AS b;";
-      DataSet ds = MySqlHelper.ExecuteDataset(Settings.ConnectionString, String.Format(selectFormat, Id));
-      foreach (DataRow detail_row in ds.Tables[0].Rows)
-      {
-        ProductPreReq productPreeequisite = new ProductPreReq(detail_row, IncomingFolder);
-        if (productPreeequisite.Generated)
-          Prerequisites.Add(productPreeequisite);
-        else
-        {
-          /*
-          if (productPreeequisite.X64 && productPreeequisite.Commercial)
-            x64COMPattern = GetRegex(productPreeequisite.RegularExpression);
-          else if (productPreeequisite.X64 && !productPreeequisite.Commercial)
-            x64GPLPattern = GetRegex(productPreeequisite.RegularExpression);
-          else if (!productPreeequisite.X64 && productPreeequisite.Commercial)
-            x86COMPattern = GetRegex(productPreeequisite.RegularExpression);
-          else if (!productPreeequisite.X64 && !productPreeequisite.Commercial)
-            x86GPLPattern = GetRegex(productPreeequisite.RegularExpression);
-           */
-        }
-      }
-
-      NumJobFilesRequired = 1;
-      if (x64GPLPattern != null) NumJobFilesRequired++;
-      if (x86COMPattern != null) NumJobFilesRequired++;
-      if (x64COMPattern != null) NumJobFilesRequired++;
+      LoadProductFiles();
+    }
+
+    private void LoadProductFiles()
+    {
+      string sql = "SELECT * FROM abs_product_files";
+      DataSet ds = MySqlHelper.ExecuteDataset(Settings.ConnectionString, sql);
+      if (ds == null || ds.Tables.Count == 0 || ds.Tables[0].Rows.Count == 0)
+        throw new Exception("Unable to load files for product");
+      foreach (DataRow row in ds.Tables[0].Rows)
+        ProductFiles.Add(new ProductFile(this, row));
     }
 
     public int Id { get; private set; }
@@ -70,12 +47,7 @@
     public string IncomingFolder { get; private set; }
     public string BuildFolder { get; private set; }
     public string PreReqsFolder { get; private set; }
-    public List<ProductPreReq> Prerequisites { get; private set; }
-
-    private Regex x86GPLPattern;
-    private Regex x64GPLPattern;
-    private Regex x86COMPattern;
-    private Regex x64COMPattern;
+    public List<ProductFile> ProductFiles { get; private set; }
 
     public static Product GetProduct(int id)
     {
@@ -93,27 +65,7 @@
       return products;
     }
 
-    private Regex GetRegex(object o)
-    {
-      if (o == null || o == DBNull.Value) return null;
-      string p = o as string;
-      return new Regex(p.ToLowerInvariant());
-    }
-
-    public bool BuildPreReqs(Version v)
-    {
-      bool allSuccess = true;
-
-      if (Prerequisites.Count == 0)
-        return allSuccess;
-
-      foreach (ProductPreReq pp in Prerequisites)
-        allSuccess &= pp.Build(v);
-
-      return allSuccess;
-    }
-
-    public void CreateJob(string[] files)
+    public void CreateJob()
     {
       Job job = null;
 
@@ -129,17 +81,15 @@
           MySqlHelper.ExecuteNonQuery(c, sql, new MySqlParameter("@product_id", Id));
           // determine the job id
           long jobId = (long)MySqlHelper.ExecuteScalar(c, "SELECT last_insert_id()");
-          job = Job.GetJob((int)jobId);          
+          job = Job.GetJob((int)jobId);
         }
 
         // move files to queue
         Directory.CreateDirectory(job.JobFolder);
-        Utility.MoveFiles(files, IncomingFolder, job.JobFolder);
-
-        // now register all the files for the job
-        foreach (string file in files)
+        foreach (ProductFile file in ProductFiles)
         {
-          job.AddFile(file, !IsCommercialFile(file), IsX64File(file));
+          Utility.MoveFile(file.File, IncomingFolder, job.JobFolder);
+          job.AddFile(file.File, file.Commercial, file.X64);
         }
 
         OwningTeam.NotifyOfJobSubmission(job);
@@ -148,73 +98,48 @@
       {
         if (job != null)
         {
-          Utility.MoveFiles(files, job.JobFolder, IncomingFolder);
+          Utility.MoveAllFiles(job.JobFolder, IncomingFolder);
           job.Delete();
         }
         throw;
       }
     }
 
-    private bool IsCommercialFile(string file)
-    {
-      if (x86COMPattern != null && x86COMPattern.Match(file).Success)
-        return true;
-      if (x64COMPattern != null && x64COMPattern.Match(file).Success)
-        return true;
-      return false;
-    }
-
-    private bool IsX64File(string file)
-    {
-      if (x64COMPattern != null && x64COMPattern.Match(file).Success)
-        return true;
-      if (x64GPLPattern != null && x64GPLPattern.Match(file).Success)
-        return true;
-      return false;
-    }
-
-    public bool ValidateFiles(string[] files)
-    {
-      // first get our x86 gpl match
-      if (x86GPLPattern == null) throw new Exception("X86 GPL pattern is required");
-      Match refMatch = null;
-
-      bool X86G = true;
-      bool X64G = x64GPLPattern != null;
-      bool X86C = x86COMPattern != null;
-      bool X64C = x64COMPattern != null;
-
-      foreach (string file in files)
-      {
-        //if (x86GPLPattern != null && x86GPLPattern.Match(file).Success) x86gpl_files.Add(file);
-        if (X86G && MatchFile(file, x86GPLPattern, ref refMatch, ref X86G)) continue;
-        if (X64G && MatchFile(file, x64GPLPattern, ref refMatch, ref X64G)) continue;
-        if (X86C && MatchFile(file, x86COMPattern, ref refMatch, ref X86C)) continue;
-        if (X64C && MatchFile(file, x64COMPattern, ref refMatch, ref X64C)) continue;
-        //if (x64GPLPattern != null && x64GPLPattern.Match(file).Success) x64gpl_files.Add(file);
-        //if (x86COMPattern != null && x86COMPattern.Match(file).Success) x86com_files.Add(file);
-        //if (x64COMPattern != null && x64COMPattern.Match(file).Success) x64com_files.Add(file);
-      }
-
-      // if any required variants were not met, then we are not complete
-      if (X86G || X64G || X86C || X64C) return false;
-      return true;
-    }
-
-    private bool MatchFile(string file, Regex pattern, ref Match refMatch, ref bool flag)
-    {
-      Match m = pattern.Match(file);
-      if (!m.Success) return false;
-      if (refMatch == null)
-        refMatch = m;
-      else
-      {
-        if (m.Groups.Count != refMatch.Groups.Count) return false;
-        for (int i = 1; i < m.Groups.Count - 1; i++)
-          if (m.Groups[i].Value != refMatch.Groups[i].Value) return false;
-      }
-      flag = false;
-      return true;
+    private void GenerateMissingFiles()
+    {
+      Version v = null;
+
+      foreach (ProductFile file in ProductFiles)
+        if (!file.Generated)
+        {
+          v = Utility.GetVersion(file.File, true);
+          break;
+        }
+
+      foreach (ProductFile file in ProductFiles)
+      {
+        if (!file.Generated) continue;
+        if (file.File != null) continue;
+        file.Build(v);
+      }
+    }
+
+    public void ValidateFiles()
+    {
+      // validate all non-generated files first
+      foreach (ProductFile file in ProductFiles)
+        file.Validate(false);
+
+      // generate any files that are set to auto-generate
+      GenerateMissingFiles();
+
+      // now see if _all_ files are present
+      foreach (ProductFile file in ProductFiles)
+        file.Validate(true);
+
+      string[] allFiles = Directory.GetFiles(Settings.IncomingFolder);
+      if (allFiles.Length != ProductFiles.Count)
+        throw new Exception("Number of files in incoming folder does not match required number of files for release");
     }
   }
 }

=== renamed file 'AbsUtility/ProductPreReq.cs' => 'AbsUtility/ProductFile.cs'
--- a/AbsUtility/ProductPreReq.cs	2012-09-14 22:32:05 +0000
+++ b/AbsUtility/ProductFile.cs	2012-09-17 20:04:19 +0000
@@ -13,51 +13,55 @@
 
 namespace Abs
 {
-  public class ProductPreReq
+  public class ProductFile
   {
-    public ProductPreReq(DataRow row, string resultFolder)
+    private string ResultFolder { get; set; }
+    private string BuildFolder { get; set; }
+    private StringBuilder outputLines;
+    private StringBuilder errorLines;
+
+    public ProductFile(Product p, DataRow row)
     {
-      Id = row["id"].ToString();
+      Id = (int)row["id"];
+      Product = p;
       RegularExpression = row["regex_pattern"].ToString();
       GenerationScript = row["generation_script"].ToString();
       X64 = row["x64"].ToString() == "1";
       Commercial = row["gpl"].ToString() == "0";
       Series = row["series"].ToString();
       SubDir = row["subdir"].ToString();
-
-      if (!Generated)
-        return;
-
-      ResultFolder = resultFolder;
-      BuildFolder = Path.Combine(resultFolder, "prereqs", SubDir, Id);
-
-      Completed = (Directory.GetFiles(ResultFolder, RegularExpression).Length > 0);
+      Generated = !String.IsNullOrEmpty(GenerationScript);
+      ResultFolder = p.IncomingFolder;
+      BuildFolder = Path.Combine(ResultFolder, "prereqs", SubDir, Id.ToString());
     }
 
+    public int Id { get; private set; }
+    public Product Product { get; private set; }
     public string RegularExpression { get; private set; }
     public string GenerationScript { get; private set; }
     public string SubDir { get; private set; }
     public bool X64 { get; private set; }
     public bool Commercial { get; private set; }
     public string Series { get; private set; }
-
-    public bool Generated { get { return !String.IsNullOrEmpty(GenerationScript); } }
-    public bool IsServer { get { return !String.IsNullOrEmpty(Series); } }
-  
-    public bool Completed { get; private set; }
-
-    private string Id { get; set; }
-    private string ResultFolder { get; set; }
-    private string BuildFolder { get; set; }
-
-    private StringBuilder outputLines;
-    private StringBuilder errorLines;
+    public bool Generated { get; private set; }
+    public string File { get; private set; }
+
+    public void Validate(bool checkGenerated)
+    {
+      if (String.IsNullOrEmpty(RegularExpression))
+        throw new Exception("Every product file requires a regular expression");
+      if (Generated && String.IsNullOrEmpty(GenerationScript))
+        throw new Exception("Generated files require a supplied generation script");
+      string[] files = Directory.GetFiles(Product.IncomingFolder, RegularExpression);
+      if (files.Length != 1 && checkGenerated)
+        throw new Exception("Regular expression did not match exactly 1 file");
+      File = files[0];
+      if (Utility.FileIsLocked(File))
+        throw new Exception(String.Format("File matching expression {0} is still locked", RegularExpression));
+    }
 
     public bool Build(Version newVersion)
     {
-      if (Completed)
-        return Completed;
-
       if (!Directory.Exists(BuildFolder))
         Directory.CreateDirectory(BuildFolder);
 
@@ -93,10 +97,7 @@
       p.Start();
       p.WaitForExit();
 
-      if (Sign())
-        Completed = (Directory.GetFiles(ResultFolder, RegularExpression).Length > 0);
-
-      return Completed;
+      return Sign();
     }
 
     public bool Sign()

=== modified file 'AbsUtility/Utility.cs'
--- a/AbsUtility/Utility.cs	2012-09-04 21:25:29 +0000
+++ b/AbsUtility/Utility.cs	2012-09-17 20:04:19 +0000
@@ -26,15 +26,30 @@
       }
     }
 
+    public static void MoveAllFiles(string srcDir, string destDir)
+    {
+      string[] files = Directory.GetFiles(srcDir);
+      foreach (string srcFile in files)
+      {
+        // Use static Path methods to extract only the file name from the path.
+        string filename = Path.GetFileName(srcFile);
+        string destFile = Path.Combine(destDir, filename);
+        File.Move(srcFile, destFile);
+      }
+    }
+
     public static void MoveFiles(string[] files, string srcDir, string destDir)
     {
       foreach (string file in files)
-      {
-        string name = Path.GetFileName(file);
-        string srcFile = Path.Combine(srcDir, name);
-        string destFile = Path.Combine(destDir, name);
-        File.Move(srcFile, destFile);
-      }
+        MoveFile(file, srcDir, destDir);
+    }
+    
+    public static void MoveFile(string file, string srcDir, string destDir)
+    {
+      string name = Path.GetFileName(file);
+      string srcFile = Path.Combine(srcDir, name);
+      string destFile = Path.Combine(destDir, name);
+      File.Move(srcFile, destFile);
     }
 
     public static string GetSeries(string filename, string defaultSeries)
@@ -110,19 +125,25 @@
       return FilesAreDoneCopying(files);
     }
 
+    public static bool FileIsLocked(string file)
+    {
+      try
+      {
+        FileStream fs = File.Open(file, FileMode.Open, FileAccess.ReadWrite);
+        fs.Close();
+        return false;
+      }
+      catch (IOException)
+      {
+        return true;
+      }
+    }
+
     private static bool FilesAreDoneCopying(string[] files)
     {
       foreach (string file in files)
       {
-        try
-        {
-          FileStream fs = File.Open(file, FileMode.Open, FileAccess.ReadWrite);
-          fs.Close();
-        }
-        catch (IOException)
-        {
-          return false;
-        }
+        if (FileIsLocked(file)) return false;
       }
       return true;
     }


Attachment: [text/bzr-bundle] bzr/reggie.burnett@oracle.com-20120917200419-9ph4bp2em595cmgb.bundle
Thread
bzr commit into ABSv2 branch (reggie.burnett:34) Reggie Burnett19 Sep