# -*- python -*-
# -*- coding: utf-8 -*-
# ex: set syntax=python:

# This is a sample buildmaster config file. It must be installed as
# 'master.cfg' in your buildmaster's base directory.

# This is the dictionary that the buildmaster pays attention to. We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}

import os, sys
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(__file__))))

from bbsupport import (ToolVersions, CompileAndShowVersion,
                       LineCount, CheckMemory, CheckSpeed, BuildTahoe,
                       BuiltTest,
                       GenCoverage, ArchiveCoverage, UploadCoverage,
                       UnarchiveCoverage,
                       TahoeVersion, CreateEgg, InstallToEgg, TestFromEgg,
                       InstallToPrefixDir, TestFromPrefixDir,
                       UploadTarballs, TestOldDep, TestAlreadyHaveDep)

####### BUILDSLAVES

from buildbot.buildslave import BuildSlave
# we tell slave admins to use master= buildmaster.tahoe-lafs.org:9987

import secrets

buildslaves = secrets.buildslaves


c['slaves'] = [BuildSlave(slavename, pw, missing_timeout=7200)
               for slavename,pw in buildslaves.items()]
c['slavePortnum'] = 9987

# the webhook sends us: https://github.com/tahoe-lafs/tahoe-lafs
# some buildslaves have problems with SSL. (? Zooko says: which ones and what
# problems?)

# REPOS can start with "git:" or "https:" and end with no suffix or with ".git"
REPOS = []

PROJECT='tahoe-lafs'
for HOST in ['github.com', 'tahoe-lafs.org']:
    for USER in ['tahoe-lafs', 'warner', 'zooko', 'davidsarah']:
        # GoodRepo forcibly replaces https: with git: (to work around some
        # buildslaves that fail to handle SSL correctly: Kyle OpenBSD and
        # FreeStorm CentOS). It checks URLs against the whitelist after this
        # substitution. So don't bother registering https: URLs.
        PATH = '/'.join([HOST, USER, PROJECT])
        REPOS.append("https://" + PATH)
        REPOS.append("https://" + PATH + ".git")
DEFAULT_REPOURL = "https://github.com/tahoe-lafs/tahoe-lafs"

from safe_repourls import GoodRepo
REPOURL = GoodRepo(REPOS, DEFAULT_REPOURL)

from buildbot.config import BuilderConfig

####### BUILDERS
from buildbot.steps.python import PyFlakes
from buildbot.process import factory
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand

def make_factory(python=None,
                 do_test_already_have_dep=True,
                 do_pyflakes_linecounts=False,
                 do_coverage=False,
                 do_test_old_dep=True,
                 do_gridchecks=[],
                 create_egg=True,
                 do_install_to_egg=False,
                 do_test_from_egg=False,
                 do_install_to_prefixdir=False,
                 do_test_from_prefixdir=False,
                 MAKE='make',
                 TAR='tar',
                 test_suite=None,
                 testtimeout=7200):

    f = factory.BuildFactory()
    f.addStep(Git(repourl=REPOURL, mode="full", clobberOnFailure=True))
    f.addStep(ToolVersions(python=python))

    # TestAlreadyHaveDep has to come before BuildTahoe so that it can
    # see whatever happens when there is not a satisfying version of
    # pycryptopp already present. (BuildTahoe will build one.)
    if do_test_already_have_dep:
        f.addStep(TestAlreadyHaveDep(python=python,
                                     haltOnFailure=False, flunkOnFailure=True,
                                     timeout=testtimeout))

    f.addStep(BuildTahoe(python=python, timeout=7200))
    f.addStep(TahoeVersion(python=python))

    if do_pyflakes_linecounts:
        f.addStep(PyFlakes(command=[MAKE, "pyflakes"],
                           warnOnWarnings=True, flunkOnFailure=True))
        f.addStep(LineCount(command=[MAKE, "count-lines"]))

    if do_coverage:
        # do not build packages if tests fail
        f.addStep(GenCoverage(python=python, haltOnFailure=True))
        f.addStep(ArchiveCoverage(TAR=TAR))
        f.addStep(UploadCoverage(upload_furlfile='../../upload-coverage.furl'))
        f.addStep(UnarchiveCoverage(unarch_furlfile='../../unarchive-coverage.furl'))
    else:
        # do not build packages if tests fail
        f.addStep(BuiltTest(python=python, test_suite=test_suite,
                            haltOnFailure=True,
                            timeout=testtimeout))

    if do_test_old_dep:
        f.addStep(TestOldDep(python=python,
                             warnOnFailure=True, flunkOnFailure=False,
                             timeout=testtimeout))

    for (name, clientdir) in do_gridchecks:
        checkcommand = [MAKE, "check-grid", "TESTCLIENTDIR=%s" % clientdir]
        f.addStep(ShellCommand(command=checkcommand, haltOnFailure=True,
                               name="check-grid-%s" % name,
                               description=["checking","against",name,"grid"],
                               descriptionDone=["check-grid", name],
                               ))

    if create_egg:
        f.addStep(CreateEgg())
    if do_install_to_egg:
        f.addStep(InstallToEgg())
        if do_test_from_egg:
            # The following step isn't done on all builders because our tests
            # take too long to run.
            egg_test_suite = 'allmydata.test.test_system'
            if test_suite:
                # if this builder uses less than the full test suite, do the
                # same for the test-from-egg: that one ARM slave is really
                # slow and can't handle the whole test_system
                egg_test_suite = test_suite
            f.addStep(TestFromEgg(testsuite=egg_test_suite,
                                  warnOnFailure=True, flunkOnFailure=False,
                                  timeout=testtimeout))

    if do_install_to_prefixdir:
        f.addStep(InstallToPrefixDir())
        # The following step isn't done on all builders because our tests
        # take too long to run or because the builder doesn't have all the
        # dependencies installed.
        if do_test_from_prefixdir:
            f.addStep(TestFromPrefixDir(testsuite='allmydata.test.test_runner',
                                        warnOnFailure=False,
                                        flunkOnFailure=False,
                                        timeout=testtimeout))

    return f

def make_tarball_factory(upload_tarballs=False, MAKE='make', TAR='tar'):
    f = factory.BuildFactory()
    f.addStep(Git(repourl=REPOURL, mode='full', clobberOnFailure=True))
    f.addStep(ShellCommand(command=[MAKE, "fetch-and-unpack-deps"],
                           name="fetch-deps",
                           description=["fetching", "dependencies"],
                           descriptionDone=["fetch", "deps"],
                           haltOnFailure=True,
                           ))
    f.addStep(ShellCommand(command=[MAKE, "tarballs"],
                           name="tarballs",
                           description=["making", "tarballs"],
                           descriptionDone=["tarballs"],
                           haltOnFailure=True,
                           ))
    if upload_tarballs:
        f.addStep(UploadTarballs(make=MAKE))
    return f

def make_clean_factory(python=None, MAKE='make', TAR='tar'):
    f = factory.BuildFactory()

    f.addStep(Git(repourl=REPOURL, mode='full', clobberOnFailure=True))
    f.addStep(ToolVersions(python=python))
    test_command = [MAKE, "test-git-ignore"]
    if python:
        test_command.append("PYTHON=%s" % python)
    f.addStep(ShellCommand(name="test-gitignore",
                           description=["testing", "gitignore"],
                           descriptionDone=["test", "gitignore"],
                           command=test_command))

    f.addStep(Git(repourl=REPOURL, mode='full', clobberOnFailure=True))
    test_command = [MAKE, "test-clean"]
    if python:
        test_command.append("PYTHON=%s" % python)
    f.addStep(ShellCommand(name="test-clean",
                           description=["testing", "clean"],
                           descriptionDone=["test", "clean"],
                           command=test_command))
    test_command = [MAKE, "test-desert-island"]
    if python:
        test_command.append("PYTHON=%s" % python)
    f.addStep(ShellCommand(name="test-desert-island",
                           description=["testing", "desert", "island"],
                           descriptionDone=["test", "desert", "island"],
                           command=test_command))
    return f

def make_memcheck_factory(platform, MAKE='make'):
    f = factory.BuildFactory()
    f.addStep(Git(repourl=REPOURL, mode='full', clobberOnFailure=True))
    build_command = [MAKE, "build"]
    f.addStep(CompileAndShowVersion(command=build_command, timeout=7200))
    assert isinstance(platform, str)
    f.addStep(CheckMemory(platform, MAKE, timeout=7200))
    return f

def make_speedcheck_factory(clientdir, linkname, MAKE='make'):
    f = factory.BuildFactory()
    f.addStep(Git(repourl=REPOURL, mode='full', clobberOnFailure=True))
    build_command = [MAKE, "build"]
    f.addStep(CompileAndShowVersion(command=build_command, timeout=7200))
    f.addStep(CheckSpeed(clientdir, linkname, MAKE))
    return f

from buildbot import locks
perfnet_lock = locks.MasterLock("perfnet")

######## BUILDERS

b_tests = b = []

#b.append(BuilderConfig(name="Dreamplug",
#                       slavenames=["jbj-dreamplug"],
#                       factory=make_factory(do_install_to_egg=False,
#                                            do_test_from_egg=False,
#                                            do_install_to_prefixdir=True,
#                                            do_test_from_prefixdir=False),
#                       category="hopeful",
#                       ))

b.append(BuilderConfig(name="Eugen lenny-amd64",
                       slavenames=["88.198.119.225"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="hopeful",
                       ))

b.append(BuilderConfig(name="Atlas ubuntu natty",
                       slavenames=["atlas1"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_old_dep=False),
                       locks=[perfnet_lock],
                       category="supported",
                       ))

if False: # needs update to buildbot-0.8.*, git exception
    b.append(BuilderConfig(name="Arthur lenny c7 32bit",
                           slavenames=["arthur-lenny-c7-32bit"],
                           factory=make_factory(do_install_to_egg=False,
                                                do_test_from_egg=False,
                                                do_install_to_prefixdir=True,
                                                do_test_from_prefixdir=False),
                           category="unsupported"
                           ))

b.append(BuilderConfig(name="Randy FreeBSD-amd64",
                       slavenames=["freebsd.psg"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            MAKE='gmake',
                                            TAR='gtar'),
                       category="offline",
                       ))

# code coverage tool temporarily broken
b.append(BuilderConfig(name="Kyle OpenBSD amd64",
                       slavenames=["starfish"],
                       factory=make_factory(do_coverage=False,
                                            do_pyflakes_linecounts=True,
                                            do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=True,
                                            MAKE='gmake',
                                            TAR='gtar'),
                       category="hopeful",
                       ))

b.append(BuilderConfig(name="MM netbsd5 i386 warp",
                       slavenames=["warp"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            MAKE='gmake',
                                            TAR='gtar'),
                       category="unsupported",
                       ))

b.append(BuilderConfig(name="Markus slackware64 current",
                       slavenames=["slackhead"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            MAKE='make',
                                            TAR='tar'),
                       category="unsupported",
                       ))

b.append(BuilderConfig(name="Markus slackware64 v14.0",
                       slavenames=["slackhorse"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            MAKE='make',
                                            TAR='tar'),
                       category="unsupported",
                       ))

b.append(BuilderConfig(name="elb debian6 ppc64",
                       slavenames=["elb-debian6-ppc64"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            MAKE='make',
                                            TAR='tar'),
                       category="unsupported",
                       ))

b.append(BuilderConfig(name="Francois lenny-armv5tel",
                       slavenames=["francois-ts109-armv5tel"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False,
                                            test_suite="allmydata.test.test_download",
                                            testtimeout=25200),
                       category="unsupported",
                       ))

b.append(BuilderConfig(name="FreeStorm CentOS5-i386",
                       slavenames=["freestorm-centos5-i386"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="hopeful",
                       ))

b.append(BuilderConfig(name="FreeStorm CentOS6-amd64",
                       slavenames=["freestorm-centos6-amd64"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="hopeful",
                       ))

b.append(BuilderConfig(name="FreeStorm WinXP-x86 py2.6",
                       slavenames=["freestorm-winxp"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="hopeful",
                       ))

if False: # freestorm says the slave is dead
    b.append(BuilderConfig(name="FreeStorm Win7-amd64-mingw py2.6",
                           slavenames=["freestorm-win7-64-mingw"],
                           factory=make_factory(do_install_to_egg=True,
                                                do_test_from_egg=True,
                                                do_install_to_prefixdir=True,
                                                do_test_from_prefixdir=False),
                           category="unsupported",
                           ))

b.append(BuilderConfig(name="Dcoder Win7-64 py2.6",
                       slavenames=["boing-win64"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="offline",
                       ))

b.append(BuilderConfig(name="Marcus Cygwin WinXP",
                       slavenames=["wanners.net-winxp-cygwin"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=True),
                       category="hopeful",
                       ))

b.append(BuilderConfig(name="sickness OpenBSD 5.0 x86 py2.7",
                       slavenames=["sickness-openbsd"],
                       factory=make_factory(do_install_to_egg=True,
                                            do_test_from_egg=True,
                                            do_install_to_prefixdir=True,
                                            do_test_from_prefixdir=False),
                       category="offline",
                       ))

b_other = []

b_other.append(BuilderConfig(name="tarballs",
                             slavenames=["atlas1"],
                             factory=make_tarball_factory(upload_tarballs=True),
                             category="supported",
                             ))

b_other.append(BuilderConfig(name="clean",
                             slavenames=["atlas1"],
                             factory=make_clean_factory(),
                             category="supported",
                             ))

b_memcheck = []
b_memcheck.append(BuilderConfig(name="memcheck-32",
                                slavenames=["atlas2"],
                                factory=make_memcheck_factory("32"),
                                category="supported",
                                ))

b_memcheck.append(BuilderConfig(name="memcheck-64",
                                slavenames=["luther2"],
                                factory=make_memcheck_factory("64"),
                                category="offline",
                                ))

b_speed = []

b_speed.append(BuilderConfig(name="speed-DSL",
                             slavenames=['luther2'],
                             factory=make_speedcheck_factory("~/tahoe-speed-client", "DSL"),
                             locks=[perfnet_lock],
                             category="supported",
                             ))

b_speed.append(BuilderConfig(name="speed-fiber",
                             slavenames=['marlowe-fiber'],
                             factory=make_speedcheck_factory("~/tahoe-speed-client", "fiber"),
                             locks=[perfnet_lock],
                             category="offline",
                             ))

b_speed.append(BuilderConfig(name="speed-colo",
                             slavenames=['atlas1'],
                             factory=make_speedcheck_factory("~/tahoe-speed-client", "colo"),
                             locks=[perfnet_lock],
                             category="supported",
                             ))

c['builders'] = b_tests + b_other + b_memcheck + b_speed

from buildbot.schedulers.basic import SingleBranchScheduler
from buildbot.schedulers.timed import Nightly
from buildbot.changes import filter
from buildbot.schedulers.forcesched import ForceScheduler

master_filter = filter.ChangeFilter(branch='master')
s_tests = SingleBranchScheduler(name="tests",
                                change_filter=master_filter,
                                treeStableTimer=2,
                                builderNames=[b.name for b in b_tests])
# we used to gate the 'tarballs' builder on successful builds of the
# "supported builders", which has been whittled down over the years to just
# "lucid-amd64" and "Atlas ubuntu natty"
s_other = SingleBranchScheduler(name="other",
                                change_filter=master_filter,
                                treeStableTimer=60,
                                builderNames=[b.name for b in b_other])
s_memcheck = SingleBranchScheduler(name="memcheck",
                                   change_filter=master_filter,
                                   treeStableTimer=300,
                                   builderNames=[b.name for b in b_memcheck])
# speed-colo currently takes about 3 minutes. It shares a lock with #
# speed-colo to make sure two simultaneous uses of perfnet don't mess up the
# results.

# This currently runs on the speedcheck-grid on atlas1. atlas1 also runs the
# 'natty' buildslave (which takes 15 minutes on each checkin), and the
# 'tarballs' builder (which takes 5 minutes on each checkin). The
# speedcheck-grid uses atlas4 as a storage server, and atlas4 is used by the
# 'clean' builder (which takes 15 minutes: most of which is
# test-desert-island), also run on each checkin.
#
# We used to run speedcheck-colo on each checkin, but getting the slavelocks
# right is too hard. So let's only run the speedcheck at night. Note that the
# buildmaster lives in UTC, so hour=10 here means 5am PDT

s_nightly_colo = Nightly('speedcheck-colo-nightly', ['speed-colo'],
                         branch="master",
                         hour=9, minute=0)
s_nightly_fiber = Nightly('speedcheck-fiber-nightly', ['speed-fiber'],
                          branch="master",
                          hour=10, minute=0)
s_nightly_dsl = Nightly('speedcheck-DSL-nightly', ['speed-DSL'],
                        branch="master",
                        hour=11, minute=0)

s_force = ForceScheduler(name="force",
                 builderNames=[ b.name for b in c['builders'] ],
                 properties=[]
                 )

c['schedulers'] = [s_tests, s_other, s_memcheck,
                   s_nightly_colo, s_nightly_fiber, s_nightly_dsl,
                   s_force ]


####### OLD BUILDERS

if False:
    f = factory.BuildFactory()
    # check out the source
    f.addStep(Git(repourl=REPOURL, mode='full'))
    # run the tests (note that this will require that 'trial' is installed)
    f.addStep(ShellCommand(command=["python", "setup.py", "build"]))
    f.addStep(ShellCommand(command=["python", "setup.py", "test"]))

    tests = []
    tests.append(
        BuilderConfig(name="FreeStorm CentOS5-i386",
                      slavenames=["freestorm-centos5-i386"],
                      #'builddir': "FreeStorm CentOS5-i386",
                      factory=make_factory(gitURL=REPOURL,
                                           do_install_to_egg=True,
                                           do_test_from_egg=True,
                                           do_install_to_prefixdir=True,
                                           do_test_from_prefixdir=False
                                           ),
                      ))
    tests.append(
        BuilderConfig(name="natty",
                      slavenames=["atlas4"],
                      #'builddir':
                      factory=make_factory(gitURL=REPOURL,
                                           do_install_to_egg=True,
                                           do_test_from_egg=True,
                                           do_install_to_prefixdir=True,
                                           do_test_from_prefixdir=False
                                           ),
                      ))
    c['builders'] = tests

    # Configure the Schedulers, which decide how to react to incoming
    # changes. In this case, just kick off a 'runtests' build

    from buildbot.schedulers.basic import SingleBranchScheduler
    from buildbot.changes import filter
    c['schedulers'] = []
    c['schedulers'].append(SingleBranchScheduler(
        name="all",
        change_filter=filter.ChangeFilter(branch='master'),
        treeStableTimer=2,
        builderNames=[b.name for b in tests]))

####### STATUS TARGETS

# 'status' is a list of Status Targets. The results of each build will be
# pushed to these targets. buildbot/status/*.py has a variety to choose from,
# including web pages, email senders, and IRC bots.

c['status'] = []

from buildbot.status import html
from buildbot.status.web import authz
authz_cfg=authz.Authz(
    # change any of these to True to enable; see the manual for more
    # options
    gracefulShutdown = True,
    forceBuild = True, # use this to test your slave once it is set up
    forceAllBuilds = True,
    pingBuilder = True,
    stopBuild = True,
    stopAllBuilds = True,
    cancelPendingBuild = True,
)
ws = html.WebStatus(http_port=8015, authz=authz_cfg)
c['status'].append(ws)

from buildbot.status import words
irc = words.IRC("irc.freenode.net", "tahoelafsbuilder",
                channels=["tahoe-lafs", "tahoe-lafs-notices"],
                password="m6z-dop6-s9",
                notify_events={ 'exception': True,
                                'successToFailure': True,
                                'failureToSuccess': True,
                                },
                allowForce=True)
c['status'].append(irc)

# MidnightMagic would like to hear when his buildslave fails
from buildbot.status.mail import MailNotifier
mn_mm = MailNotifier(fromaddr="buildmaster-tahoe@tahoe-lafs.org",
                     sendToInterestedUsers=False,
                     extraRecipients=["buildslave-notify@quake.ca"],
                     mode="failing",
                     subject="tahoe buildbot failed",
                     builders=["MM netbsd5 i386 warp"])
c['status'].append(mn_mm)


####### CHANGESOURCES
import github_posthook
github_posthook.setup(c, ws, "github_hook")

####### PROJECT IDENTITY

# the 'title' string will appear at the top of this buildbot
# installation's html.WebStatus home page (linked to the
# 'titleURL') and is embedded in the title of the waterfall HTML page.

c['title'] = "Tahoe-LAFS"
c['titleURL'] = "https://tahoe-lafs.org/trac/tahoe-lafs"

# the 'buildbotURL' string should point to the location where the buildbot's
# internal web server (usually the html.WebStatus page) is visible. This
# typically uses the port number set in the Waterfall 'status' entry, but
# with an externally-visible host name which the buildbot cannot figure out
# without some help.

c['buildbotURL'] = "https://tahoe-lafs.org/buildbot-tahoe-lafs/"

####### DB URL

# This specifies what database buildbot uses to store change and scheduler
# state.  You can leave this at its default for all but the largest
# installations.
c['db_url'] = "sqlite:///state.sqlite"

