update ucontext factory to download the archive instead of using svn; use only one...

[simgrid.git] / build / buildbot / master / master.cfg

diff --git a/build/buildbot/master/master.cfg b/build/buildbot/master/master.cfg
index fb83b5d..81b87cf 100644 (file)
--- a/build/buildbot/master/master.cfg
+++ b/build/buildbot/master/master.cfg
@@ -18,10 +18,8 @@ c['projectURL']= "http://simgrid.gforge.inria.fr/"
 c['buildbotURL'] = "http://bob.loria.fr:8010/"
 
 # Gets bot from account module (hide passphrases)
 c['buildbotURL'] = "http://bob.loria.fr:8010/"
 
 # Gets bot from account module (hide passphrases)

-from account import bots
-c['bots'] = bots
-from account import all_slaves
-
+from account import slaves,all_slaves
+c['slaves'] = slaves

 
 # Port number used by slaves
 c['slavePortnum'] = 9989
 
 # Port number used by slaves
 c['slavePortnum'] = 9989


@@ -33,10 +31,11 @@ c['slavePortnum'] = 9989
 
 from extensions import CustomSVN, CustomConfigure, CustomCheck
 from buildbot.process import factory
 
 from extensions import CustomSVN, CustomConfigure, CustomCheck
 from buildbot.process import factory

-from buildbot.steps.transfer import FileUpload
+from buildbot.steps.transfer import FileUpload,FileDownload

 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.source import SVN
 from buildbot.steps.shell import ShellCommand
 from buildbot.steps.source import SVN

-
+from buildbot.process.properties import WithProperties
+     

 # build tarball factory
 dist_factory = factory.BuildFactory()
 dist_factory.addStep(CustomSVN, name ="svn update", baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
 # build tarball factory
 dist_factory = factory.BuildFactory()
 dist_factory.addStep(CustomSVN, name ="svn update", baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")


@@ -48,40 +47,38 @@ dist_factory.addStep(ShellCommand,name = "bootstrap",description = "bootstraping
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./bootstrap"])
 dist_factory.addStep(ShellCommand,name = "configure", description="configuring",   descriptionDone="configure",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--enable-maintainer-mode"])
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./bootstrap"])
 dist_factory.addStep(ShellCommand,name = "configure", description="configuring",   descriptionDone="configure",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--enable-maintainer-mode"])

-#dist_factory.addStep(ShellCommand,name = "make clean",description = "running make clean",descriptionDone = "make clean",
-#                     environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make", "clean"])

 dist_factory.addStep(ShellCommand,name = "make -C src", description = "building src",  descriptionDone = "make -C src",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","src"])
 dist_factory.addStep(ShellCommand,name = "make -C tools", description = "building tools",  descriptionDone = "make -C tools",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","tools"])
 dist_factory.addStep(ShellCommand,name = "make dist", description = "building archive",  descriptionDone = "make dist",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","dist"])
 dist_factory.addStep(ShellCommand,name = "make -C src", description = "building src",  descriptionDone = "make -C src",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","src"])
 dist_factory.addStep(ShellCommand,name = "make -C tools", description = "building tools",  descriptionDone = "make -C tools",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4","-C","tools"])
 dist_factory.addStep(ShellCommand,name = "make dist", description = "building archive",  descriptionDone = "make dist",
                      environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","dist"])

-dist_factory.addStep(ShellCommand,name = "Prepare archive",descriptionDone="Prepare archive", haltOnFailure = True,command=["bash","-c","ln -s simgrid-*.tar.gz simgrid-svn-lastest.tar.gz"])
-dist_factory.addStep(FileUpload(name="Upload archive",slavesrc="simgrid-svn-lastest.tar.gz",masterdest="simgrid-svn-lastest.tar.gz",haltOnFailure = True))
+dist_factory.addStep(FileUpload(name="Upload archive",haltOnFailure = True,
+                                slavesrc= WithProperties("simgrid-3.3-svn-r%sM.tar.gz", "got_revision"),
+                                masterdest=WithProperties("simgrid-3.3-svn-r%sM.tar.gz", "got_revision")))

 
 

+archive = "simgrid-3.3-svn-r%(revision:-)sM.tar.gz"

 # Test tarball factories
 pthreads_factory_O3= factory.BuildFactory()
 # Test tarball factories
 pthreads_factory_O3= factory.BuildFactory()

-pthreads_factory_O3.addStep(CustomSVN, name ="{svn update}", baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
-pthreads_factory_O3.addStep(ShellCommand,name = "{svn revert}",description = "running svn revert",descriptionDone = "svn revert",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["svn" ,"revert","-R","."])
-pthreads_factory_O3.addStep(ShellCommand,name = "{configure}", description="running configure",   descriptionDone="configure",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure",  "--with-pthread","--enable-compile-warnings","--enable-compile-optimizations","--enable-botbuild"])
-pthreads_factory_O3.addStep(ShellCommand,name = "{make clean}",description = "running make clean",descriptionDone = "make clean",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make", "clean"])
-pthreads_factory_O3.addStep(ShellCommand,name = "{make}",      description = "running make",      descriptionDone = "make",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","4"])
-pthreads_factory_O3.addStep(CustomCheck,name ="{check all}",        description = "running check all", descriptionDone ="check all",
+pthreads_factory_O3.addStep(ShellCommand,name="cleanup",descriptionDone="cleanup",command=["bash","-c","rm -rf * .svn"])
+pthreads_factory_O3.addStep(FileDownload(name="get archive",haltOnFailure=True,slavedest=WithProperties(archive),mastersrc=WithProperties(archive)))
+pthreads_factory_O3.addStep(ShellCommand, name ="open archive",descriptionDone="open archive", command=["tar","xvfz",WithProperties(archive),"--strip","1"])
+pthreads_factory_O3.addStep(ShellCommand,name = "configure", description="running configure",   descriptionDone="configure",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure",  "--with-pthread","--enable-compile-warnings","--enable-compile-optimizations"])
+pthreads_factory_O3.addStep(ShellCommand,name = "make",      description = "running make",      descriptionDone = "make",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","10"])
+pthreads_factory_O3.addStep(CustomCheck,name ="check all",        description = "running check all", descriptionDone ="check all",

                             haltOnFailure = True,command=["./checkall"])
                             
 ucontext_factory_O3= factory.BuildFactory()
                             haltOnFailure = True,command=["./checkall"])
                             
 ucontext_factory_O3= factory.BuildFactory()

-ucontext_factory_O3.addStep(CustomSVN, name ="{svn update}",baseURL='svn://scm.gforge.inria.fr/svn/simgrid/simgrid/', defaultBranch='trunk', mode="update")
-ucontext_factory_O3.addStep(ShellCommand,name = "{svn revert}",description = "running svn revert",descriptionDone = "svn revert",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["svn" ,"revert","-R","."])
-ucontext_factory_O3.addStep(CustomConfigure,name ="{configure}",description="running configure",descriptionDone="configure",
+ucontext_factory_O3.addStep(ShellCommand,name="cleanup",descriptionDone="cleanup",command=["bash","-c","rm -rf * .svn"])
+ucontext_factory_O3.addStep(FileDownload(name="get archive",haltOnFailure=True,slavedest=WithProperties(archive),mastersrc=WithProperties(archive)))
+ucontext_factory_O3.addStep(ShellCommand, name ="open archive",descriptionDone="open archive", command=["tar","xvfz",WithProperties(archive),"--strip","1"])
+ucontext_factory_O3.addStep(CustomConfigure,name ="configure",description="running configure",descriptionDone="configure",

                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--with-context=ucontext","--enable-compile-warnings","--enable-compile-optimizations","--enable-botbuild"]) # Main difference with pthread_factory
                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./configure", "--with-context=ucontext","--enable-compile-warnings","--enable-compile-optimizations","--enable-botbuild"]) # Main difference with pthread_factory

-ucontext_factory_O3.addStep(ShellCommand,name = "{make}",description = "running make",descriptionDone = "make",
-                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make"])
-ucontext_factory_O3.addStep(CustomCheck,name = "{check all}",description = "running check all",descriptionDone ="check all",
+ucontext_factory_O3.addStep(ShellCommand,name = "make",description = "running make",descriptionDone = "make",
+                            environ={"LC_ALL":"C"}, haltOnFailure = True,command=["make","-j","10"])
+ucontext_factory_O3.addStep(CustomCheck,name = "check all",description = "running check all",descriptionDone ="check all",

                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./checkall"])
 
 
                             environ={"LC_ALL":"C"}, haltOnFailure = True,command=["./checkall"])
 
 


@@ -100,21 +97,19 @@ fastnet_lock = locks.MasterLock("fastnet_lock")
 # builders
 c['builders'] = [    
      {'name':'distBuilder',
 # builders
 c['builders'] = [    
      {'name':'distBuilder',

-     'slavename':'bob_dist_worker',
+     'slavename':'bob_dist',

      'builddir':'/var/lib/buildbot/simgrid/dist-worker',
      'factory':dist_factory},
      
      {'name':'linux_amd64_pthreads_O3',
      'builddir':'/var/lib/buildbot/simgrid/dist-worker',
      'factory':dist_factory},
      
      {'name':'linux_amd64_pthreads_O3',

-     'slavename':'bob_pthreads',
+     'slavename':'bob',

      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_pthreads/builddir_O3',
      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_pthreads/builddir_O3',

-     'factory':pthreads_factory_O3,
-     'locks': [bob_lock]},
+     'factory':pthreads_factory_O3},

        
      {'name':'linux_amd64_ucontext_O3',
        
      {'name':'linux_amd64_ucontext_O3',

-     'slavename':'bob_ucontext',
+     'slavename':'bob',

      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_ucontext/builddir_O3',
      'builddir':'/var/lib/buildbot/simgrid/linux_amd64_ucontext/builddir_O3',

-     'factory':ucontext_factory_O3,
-     'locks': [bob_lock]},
+     'factory':ucontext_factory_O3},

      
      {'name':'linux_i386_pthreads_O3',
      'slavename':'artimon_pthreads',
      
      {'name':'linux_i386_pthreads_O3',
      'slavename':'artimon_pthreads',


@@ -155,12 +150,12 @@ c['builders'] = [
 
 # Get changes from SVN, by polling every 60 seconds
 from buildbot.changes.svnpoller import SVNPoller
 
 # Get changes from SVN, by polling every 60 seconds
 from buildbot.changes.svnpoller import SVNPoller

-c['change_source'] = SVNPoller("svn://scm.gforge.inria.fr/svn/simgrid/simgrid/trunk", pollinterval=60)
+c['change_source'] = SVNPoller("svn://scm.gforge.inria.fr/svn/simgrid/simgrid/trunk", pollinterval=600)

 
 # Once we have a change, build a tarball
 
 from buildbot import scheduler
 
 # Once we have a change, build a tarball
 
 from buildbot import scheduler

-maketarball = scheduler.Scheduler(name="maketarball", branch=None, treeStableTimer=1, builderNames=['distBuilder'])
+maketarball = scheduler.Scheduler(name="maketarball", branch=None, treeStableTimer=60, builderNames=['distBuilder'])

 
 c['schedulers'] = [maketarball]
 
 
 c['schedulers'] = [maketarball]
 


@@ -169,15 +164,18 @@ c['schedulers'] = [maketarball]
 
 c['status'] = []
 
 
 c['status'] = []
 

+# TODO: move to modernity here (but write an index.html for that)
+#from buildbot.status.html import WebStatus
+#c['status'].append(WebStatus(8010))

 from buildbot.status import html
 c['status'].append( html.Waterfall(http_port=8010) )
 
 from buildbot.status import words                                       
 from buildbot.status import html
 c['status'].append( html.Waterfall(http_port=8010) )
 
 from buildbot.status import words                                       

-c['status'].append(words.IRC(host="irc.oftc.net", nick="buildbot", channels=["#simgrid"]))
+c['status'].append(words.IRC(host="irc.oftc.net", nick="simgrid_buildbot", channels=["#simgrid"]))

 
 from buildbot.status.mail import MailNotifier
 
 from buildbot.status.mail import MailNotifier

-mn = MailNotifier(fromaddr="SimGrid buildbot <sg-dev@lists.gforge.inria.fr>",
-                  sendToInterestedUsers=False, extraRecipients=['martin.quinson@loria.fr'])
+mn = MailNotifier(fromaddr="sg-dev@lists.gforge.inria.fr",
+                  sendToInterestedUsers=False, addLogs=True, extraRecipients=['martin.quinson@loria.fr'])

 c['status'].append(mn)
 
 ####################################################################################
 c['status'].append(mn)
 
 ####################################################################################


@@ -185,12 +183,10 @@ c['status'].append(mn)
 
 # We use only one scheduling right now, a nightly one, which reruns everything
 
 
 # We use only one scheduling right now, a nightly one, which reruns everything
 

-# TODO: setup a rebuilder in response to SVN commits

 # TODO: Define a list of all slaves and use it here and in the definition of c['builders']
 
 from buildbot.scheduler import Scheduler, Nightly, Periodic
 
 # TODO: Define a list of all slaves and use it here and in the definition of c['builders']
 
 from buildbot.scheduler import Scheduler, Nightly, Periodic
 

-

 #nightly_scheduler = Nightly(
 #       "nightly",    # name
 #       all_slaves,   # defined in account.py
 #nightly_scheduler = Nightly(
 #       "nightly",    # name
 #       all_slaves,   # defined in account.py


@@ -200,6 +196,6 @@ from buildbot.scheduler import Scheduler, Nightly, Periodic
 #hourly = Periodic("hourly_scheduler", all_slaves, 60*60*2) # delay in seconds
 
 #c['schedulers'] = [nightly_scheduler,hourly]
 #hourly = Periodic("hourly_scheduler", all_slaves, 60*60*2) # delay in seconds
 
 #c['schedulers'] = [nightly_scheduler,hourly]

-c['schedulers'] = []
+#c['schedulers'] = []