Logo AND Algorithmique Numérique Distribuée

Public GIT Repository
update ucontext factory to download the archive instead of using svn; use only one...
[simgrid.git] / build / buildbot / master / master.cfg
index 4a904c1..81b87cf 100644 (file)
 
 c = BuildmasterConfig = {}
 
-# Gets bot from account module
-from account import bots
-c['bots'] = bots
+# Project informations
+
+c['projectName'] = "SimGrid compilation status"
+c['projectURL']= "http://simgrid.gforge.inria.fr/"
+c['buildbotURL'] = "http://bob.loria.fr:8010/"
 
+# Gets bot from account module (hide passphrases)
+from account import slaves,all_slaves
+c['slaves'] = slaves
 
 # Port number used by slaves
 c['slavePortnum'] = 9989
 
 
-####### CHANGESOURCES
-
-# the 'sources' list tells the buildmaster how it should find out about
-# source code changes. Any class which implements IChangeSource can be added
-# to this list: there are several in buildbot/changes/*.py to choose from.
-
-c['sources'] = []
-
-# For example, if you had CVSToys installed on your repository, and your
-# CVSROOT/freshcfg file had an entry like this:
-#pb = ConfigurationSet([
-#    (None, None, None, PBService(userpass=('foo', 'bar'), port=4519)),
-#    ])
-
-# then you could use the following buildmaster Change Source to subscribe to
-# the FreshCVS daemon and be notified on every commit:
-#
-#from buildbot.changes.freshcvs import FreshCVSSource
-#fc_source = FreshCVSSource("cvs.example.com", 4519, "foo", "bar")
-#c['sources'].append(fc_source)
-
-# or, use a PBChangeSource, and then have your repository's commit script run
-# 'buildbot sendchange', or contrib/svn_buildbot.py, or
-# contrib/arch_buildbot.py :
-#
-#from buildbot.changes.pb import PBChangeSource
-#c['sources'].append(PBChangeSource())
-
-
-
-####################################################################################
-## Scheduler configuration
-
-# We use only one scheduling right now, a nightly one, which reruns everything
-
-# TODO: setup a rebuilder in response to SVN commits
-# TODO: Define a list of all slaves and use it here and in the definition of c['builders']
-
-from buildbot.scheduler import Scheduler, Nightly, Periodic
-
-nightly_scheduler = Nightly(
-       "nightly", 
-       ["linux_amd64_pthreads_O3",
-       "linux_amd64_ucontext_O3",
-       "linux_i386_pthreads_O3",
-       "linux_i386_ucontext_O3",
-       "mac_os_x_pthreads",
-       "mac_os_x_ucontext",
-       "windows_builder"], 
-       hour=[12,24], 
-       minute=15)          
-c['schedulers'] = [nightly_scheduler]
-
-now = Periodic("now", [
-      "linux_amd64_pthreads_O3",
-      "linux_amd64_ucontext_O3",
-      "linux_i386_pthreads_O3",
-      "linux_i386_ucontext_O3",
-      "mac_os_x_pthreads",
-      "mac_os_x_ucontext",
-      "windows_builder"
-      ], 60*20*1)
-c['schedulers'] = [nightly_scheduler]
-
-
 
 ####################################################################################
 # builders declarations
 
 from extensions import CustomSVN, CustomConfigure, CustomCheck
-from buildbot.process import step, factory
-from buildbot.process.step import ShellCommand
-
-# factories
+from buildbot.process import factory
+from buildbot.steps.transfer import FileUpload,FileDownload
+from buildbot.steps.shell import ShellCommand
+from buildbot.steps.source import SVN
+from buildbot.process.properties import WithProperties
+     
+# build tarball factory
+dist_factory = factory.BuildFactory()
+dist_factory.addStep(CustomSVN, name ="svn update", baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
+dist_factory.addStep(ShellCommand,name ="clean oldies",description = "removing old archives", descriptionDone ="clean oldies",
+                     flunkOnFailure=False,command=["bash","-c","rm -rfv simgrid-*-svn-r* *.tar.gz build/*tar.gz"])
+dist_factory.addStep(ShellCommand,name = "svn revert",description = "reverting the SVN",descriptionDone = "svn revert",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["svn" ,"revert","-R","."])
+dist_factory.addStep(ShellCommand,name = "bootstrap",description = "bootstraping",descriptionDone = "bootstrap",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./bootstrap"])
+dist_factory.addStep(ShellCommand,name = "configure", description="configuring",   descriptionDone="configure",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--enable-maintainer-mode"])
+dist_factory.addStep(ShellCommand,name = "make -C src", description = "building src",  descriptionDone = "make -C src",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","src"])
+dist_factory.addStep(ShellCommand,name = "make -C tools", description = "building tools",  descriptionDone = "make -C tools",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","tools"])
+dist_factory.addStep(ShellCommand,name = "make dist", description = "building archive",  descriptionDone = "make dist",
+                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","dist"])
+dist_factory.addStep(FileUpload(name="Upload archive",haltOnFailure = True,
+                                slavesrc= WithProperties("simgrid-3.3-svn-r%sM.tar.gz", "got_revision"),
+                                masterdest=WithProperties("simgrid-3.3-svn-r%sM.tar.gz", "got_revision")))
+
+archive = "simgrid-3.3-svn-r%(revision:-)sM.tar.gz"
+# Test tarball factories
 pthreads_factory_O3= factory.BuildFactory()
-pthreads_factory_O3.addStep(CustomSVN, name ="{svn update}", baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
-pthreads_factory_O3.addStep(step.ShellCommand,name = "{svn revert}",description = "running svn revert",descriptionDone = "svn revert",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["svn" ,"revert","-R","build"])
-pthreads_factory_O3.addStep(step.ShellCommand,name = "{configure}", description="running configure",   descriptionDone="configure",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure",  "--with-pthread","--enable-compile-warnings","--enable-compile-optimizations","--enable-botbuild"])
-pthreads_factory_O3.addStep(step.ShellCommand,name = "{make clean}",description = "running make clean",descriptionDone = "make clean",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make", "clean"])
-pthreads_factory_O3.addStep(step.ShellCommand,name = "{make}",      description = "running make",      descriptionDone = "make",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4"])
-pthreads_factory_O3.addStep(CustomCheck,name ="{check all}",        description = "running check all", descriptionDone ="check all",
+pthreads_factory_O3.addStep(ShellCommand,name="cleanup",descriptionDone="cleanup",command=["bash","-c","rm -rf * .svn"])
+pthreads_factory_O3.addStep(FileDownload(name="get archive",haltOnFailure=True,slavedest=WithProperties(archive),mastersrc=WithProperties(archive)))
+pthreads_factory_O3.addStep(ShellCommand, name ="open archive",descriptionDone="open archive", command=["tar","xvfz",WithProperties(archive),"--strip","1"])
+pthreads_factory_O3.addStep(ShellCommand,name = "configure", description="running configure",   descriptionDone="configure",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure",  "--with-pthread","--enable-compile-warnings","--enable-compile-optimizations"])
+pthreads_factory_O3.addStep(ShellCommand,name = "make",      description = "running make",      descriptionDone = "make",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","10"])
+pthreads_factory_O3.addStep(CustomCheck,name ="check all",        description = "running check all", descriptionDone ="check all",
                             haltOnFailure = True,command=["./checkall"])
-
+                            
 ucontext_factory_O3= factory.BuildFactory()
-ucontext_factory_O3.addStep(CustomSVN, name ="{svn update}",baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
-ucontext_factory_O3.addStep(step.ShellCommand,name = "{svn revert}",description = "running svn revert",descriptionDone = "svn revert",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["svn" ,"revert","-R","build"])
-ucontext_factory_O3.addStep(CustomConfigure,name ="{configure}",description="running configure",descriptionDone="configure",
+ucontext_factory_O3.addStep(ShellCommand,name="cleanup",descriptionDone="cleanup",command=["bash","-c","rm -rf * .svn"])
+ucontext_factory_O3.addStep(FileDownload(name="get archive",haltOnFailure=True,slavedest=WithProperties(archive),mastersrc=WithProperties(archive)))
+ucontext_factory_O3.addStep(ShellCommand, name ="open archive",descriptionDone="open archive", command=["tar","xvfz",WithProperties(archive),"--strip","1"])
+ucontext_factory_O3.addStep(CustomConfigure,name ="configure",description="running configure",descriptionDone="configure",
                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--with-context=ucontext","--enable-compile-warnings","--enable-compile-optimizations","--enable-botbuild"]) # Main difference with pthread_factory
-ucontext_factory_O3.addStep(step.ShellCommand,name = "{make}",description = "running make",descriptionDone = "make",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make"])
-ucontext_factory_O3.addStep(CustomCheck,name = "{check all}",description = "running check all",descriptionDone ="check all",
+ucontext_factory_O3.addStep(ShellCommand,name = "make",description = "running make",descriptionDone = "make",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","10"])
+ucontext_factory_O3.addStep(CustomCheck,name = "check all",description = "running check all",descriptionDone ="check all",
                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./checkall"])
 
 
 windows_factory= factory.BuildFactory()
-windows_factory.addStep(step.SVN, name ="{svn update}",baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
-windows_factory.addStep(step.ShellCommand,name = "{make}",description = "running make",descriptionDone = "make",haltOnFailure = True,command=["C:\\buildslave\\projects\\simgrid\\builddir\\buildMake", "C:\\buildslave\\projects\\simgrid\\builddir\\make_all.tst"])
-windows_factory.addStep(step.ShellCommand,name = "{test suite}",description= "running test suite",descriptionDone ="test suite",haltOnFailure = True,command=["C:\\buildslave\\projects\\simgrid\\builddir\\Test", "C:\\buildslave\\projects\\simgrid\\builddir\\test_all.tst"])
+windows_factory.addStep(SVN, name ="{svn update}",baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
+windows_factory.addStep(ShellCommand,name = "{make}",description = "running make",descriptionDone = "make",haltOnFailure = True,command=["C:\\buildslave\\projects\\simgrid\\builddir\\buildMake", "C:\\buildslave\\projects\\simgrid\\builddir\\make_all.tst"])
+windows_factory.addStep(ShellCommand,name = "{test suite}",description= "running test suite",descriptionDone ="test suite",haltOnFailure = True,command=["C:\\buildslave\\projects\\simgrid\\builddir\\Test", "C:\\buildslave\\projects\\simgrid\\builddir\\test_all.tst"])
 
 
 # One lock per machine since some tests open sockets on fixed ports
@@ -133,19 +96,20 @@ fastnet_lock = locks.MasterLock("fastnet_lock")
 
 # builders
 c['builders'] = [    
+     {'name':'distBuilder',
+     'slavename':'bob_dist',
+     'builddir':'/var/lib/buildbot/simgrid/dist-worker',
+     'factory':dist_factory},
+     
      {'name':'linux_amd64_pthreads_O3',
-     'slavename':'bob_pthreads',
+     'slavename':'bob',
      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_pthreads/builddir_O3',
-     'factory':pthreads_factory_O3,
-     'locks': [bob_lock]},
+     'factory':pthreads_factory_O3},
        
      {'name':'linux_amd64_ucontext_O3',
-     'slavename':'bob_ucontext',
+     'slavename':'bob',
      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_ucontext/builddir_O3',
-     'factory':ucontext_factory_O3,
-     'locks': [bob_lock]},
-
-
+     'factory':ucontext_factory_O3},
      
      {'name':'linux_i386_pthreads_O3',
      'slavename':'artimon_pthreads',
@@ -182,114 +146,56 @@ c['builders'] = [
      ]
 
 
-# status html page
+####### CHANGESOURCES
+
+# Get changes from SVN, by polling every 60 seconds
+from buildbot.changes.svnpoller import SVNPoller
+c['change_source'] = SVNPoller("svn://scm.gforge.inria.fr/svn/simgrid/simgrid/trunk", pollinterval=600)
+
+# Once we have a change, build a tarball
+
+from buildbot import scheduler
+maketarball = scheduler.Scheduler(name="maketarball", branch=None, treeStableTimer=60, builderNames=['distBuilder'])
+
+c['schedulers'] = [maketarball]
+
+
+# status targets
 
 c['status'] = []
 
+# TODO: move to modernity here (but write an index.html for that)
+#from buildbot.status.html import WebStatus
+#c['status'].append(WebStatus(8010))
 from buildbot.status import html
-w = html.Waterfall(http_port=8010)
-
-c['status'].append(w)
-
-# status smtp request
-
-from buildbot.status import mail
-
-c['status'].append(mail.MailNotifier(builders=['mac_os_x_pthreads'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during pthread build of SimGrid on fastnet",
-  extraRecipients=["malek.cherier@loria.fr","martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-
-c['status'].append(mail.MailNotifier(builders=['mac_os_x_ucontext'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during ucontext build of SimGrid on fastnet",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-
-
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_pthreads'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during pthread build of SimGrid on bob",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_pthreads_O0'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during pthread build of SimGrid on bob (compiler optimizations O0 enabled)",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_pthreads_O3'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during pthread build of SimGrid on bob (compiler optimizations O3 enabled)",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_ucontext'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during ucontext build of SimGrid on bob",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))  
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_ucontext_O0'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing (compiler optimizations O0 enabled)",
-  subject = "Error occured during ucontext build of SimGrid on bob",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))  
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_amd64_ucontext_O3'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "Error occured during ucontext build of SimGrid on bob  (compiler optimizations O3 enabled)",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))  
-                                     
-c['status'].append(mail.MailNotifier(builders=['windows_builder'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on Windows platform",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_pthreads'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_pthreads_O0'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform (compiler optimizations O0 enabled)" ,
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_pthreads_O3'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform (compiler optimizations O3 enabled)" ,
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_ucontext'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_ucontext_O0'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform (compiler optimizations O0 enabled)",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))
-                                      
-c['status'].append(mail.MailNotifier(builders=['linux_i386_ucontext_O3'],
-  fromaddr="bob@loria.fr",relayhost="smtp.loria.fr",mode="failing",
-  subject = "An error occurs during the build of SimGRID on artimon platform (compiler optimizations O3 enabled)",
-  extraRecipients=["malek.cherier@loria.fr", "martin.quinson@loria.fr"],
-  sendToInterestedUsers=True))                                                                            
+c['status'].append( html.Waterfall(http_port=8010) )
+
+from buildbot.status import words                                       
+c['status'].append(words.IRC(host="irc.oftc.net", nick="simgrid_buildbot", channels=["#simgrid"]))
+
+from buildbot.status.mail import MailNotifier
+mn = MailNotifier(fromaddr="sg-dev@lists.gforge.inria.fr",
+                  sendToInterestedUsers=False, addLogs=True, extraRecipients=['martin.quinson@loria.fr'])
+c['status'].append(mn)
+
+####################################################################################
+## Scheduler configuration
+
+# We use only one scheduling right now, a nightly one, which reruns everything
+
+# TODO: Define a list of all slaves and use it here and in the definition of c['builders']
+
+from buildbot.scheduler import Scheduler, Nightly, Periodic
+
+#nightly_scheduler = Nightly(
+#       "nightly",    # name
+#       all_slaves,   # defined in account.py
+#       hour=[12,24], # schedule time
+#       minute=15)          
+
+#hourly = Periodic("hourly_scheduler", all_slaves, 60*60*2) # delay in seconds
+
+#c['schedulers'] = [nightly_scheduler,hourly]
+#c['schedulers'] = []
 
-# Project informations
 
-c['projectName'] = "SimGrid compilation status"
-c['projectURL']= "http://simgrid.gforge.inria.fr/"
-c['buildbotURL'] = "http://bob.loria.fr:8010/"