builder: fx-team-linux64-asan-debug
slave: bld-linux64-spot-551
starttime: 1445637640.73
results: exception (4)
buildid: 20151023150037
builduid: 478247e970f14a05aaf582c1ab0cd055
revision: dd5df13fccbdbca6a45ad11120a9485dc3272512

========= Started set props: master (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.734073) =========
master: http://buildbot-master91.bb.releng.usw2.mozilla.com:8001/
========= Finished set props: master (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.734608) =========

========= Started set props: basedir (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.734903) =========
bash -c pwd
 in dir /builds/slave/fx-team-l64-asan-d-00000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', 'pwd']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-551.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-asan-d-00000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
/builds/slave/fx-team-l64-asan-d-00000000000
program finished with exit code 0
elapsedTime=0.003702
basedir: '/builds/slave/fx-team-l64-asan-d-00000000000'
========= Finished set props: basedir (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.803963) =========

========= Started downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.804255) =========
========= Finished downloading to buildprops.json (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.817622) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.817882) =========
rm -rf properties
 in dir /builds/slave/fx-team-l64-asan-d-00000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'properties']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-551.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-asan-d-00000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.002434
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.865553) =========

========= Started set props: script_repo_url (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.865827) =========
script_repo_url: https://hg.mozilla.org/build/mozharness
========= Finished set props: script_repo_url (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.866159) =========

========= Started 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.866457) =========
rm -rf scripts
 in dir /builds/slave/fx-team-l64-asan-d-00000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-rf', 'scripts']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-551.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-asan-d-00000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.002323
========= Finished 'rm -rf ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:00:40.914550) =========

========= Started 'bash -c ...' (results: 0, elapsed: 3 mins, 54 secs) (at 2015-10-23 15:00:40.914864) =========
bash -c 'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev dd5df13fccbdbca6a45ad11120a9485dc3272512 --destination scripts --debug'
 in dir /builds/slave/fx-team-l64-asan-d-00000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['bash', '-c', u'python /tools/checkouts/build-tools/buildfarm/utils/archiver_client.py mozharness --repo integration/fx-team --rev dd5df13fccbdbca6a45ad11120a9485dc3272512 --destination scripts --debug']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-551.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-asan-d-00000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
2015-10-23 15:00:40,963 truncating revision to first 12 chars
2015-10-23 15:00:40,963 Setting DEBUG logging.
2015-10-23 15:00:40,963 attempt 1/10
2015-10-23 15:00:40,963 Getting archive location from https://api.pub.build.mozilla.org/archiver/hgmo/integration/fx-team/dd5df13fccbd?&preferred_region=us-west-2&suffix=tar.gz&subdir=testing/mozharness
2015-10-23 15:00:42,079 attempt 1/10
2015-10-23 15:00:42,439 current task status: no status available at this point. state: PENDING
2015-10-23 15:00:42,440 sleeping for 9.00s (attempt 1/10)
2015-10-23 15:00:51,449 attempt 2/10
2015-10-23 15:00:51,808 current task status: no status available at this point. state: PENDING
2015-10-23 15:00:51,808 sleeping for 12.50s (attempt 2/10)
2015-10-23 15:01:04,321 attempt 3/10
2015-10-23 15:01:04,676 current task status: no status available at this point. state: PENDING
2015-10-23 15:01:04,676 sleeping for 19.75s (attempt 3/10)
2015-10-23 15:01:24,446 attempt 4/10
2015-10-23 15:01:24,822 current task status: no status available at this point. state: PENDING
2015-10-23 15:01:24,822 sleeping for 28.62s (attempt 4/10)
2015-10-23 15:01:53,476 attempt 5/10
2015-10-23 15:01:53,811 current task status: no status available at this point. state: PENDING
2015-10-23 15:01:53,811 sleeping for 41.94s (attempt 5/10)
2015-10-23 15:02:35,791 attempt 6/10
2015-10-23 15:02:36,137 current task status: no status available at this point. state: PENDING
2015-10-23 15:02:36,138 sleeping for 62.91s (attempt 6/10)
2015-10-23 15:03:39,107 attempt 7/10
2015-10-23 15:03:39,264 current task status: Task completed! Check 's3_urls' for upload locations. state: SUCCESS
2015-10-23 15:03:39,330 unpacking tar archive at: fx-team-dd5df13fccbd/testing/mozharness/
program finished with exit code 0
elapsedTime=178.492612
========= Finished 'bash -c ...' (results: 0, elapsed: 3 mins, 54 secs) (at 2015-10-23 15:04:34.928068) =========

========= Started download exception (results: 4, elapsed: 30 secs) (at 2015-10-23 15:04:34.928357) =========

<style type="text/css">
    p.error {
      color: red;
      font-family: Verdana, Arial, helvetica, sans-serif;
      font-weight: bold;
    }

    div {
      font-family: Verdana, Arial, helvetica, sans-serif;
    }

    div.stackTrace {
    }

    div.frame {
      padding: 1em;
      background: white;
      border-bottom: thin black dashed;
    }

    div.firstFrame {
      padding: 1em;
      background: white;
      border-top: thin black dashed;
      border-bottom: thin black dashed;
    }

    div.location {
    }

    div.snippet {
      margin-bottom: 0.5em;
      margin-left: 1em;
      background: #FFFFDD;
    }

    div.snippetHighlightLine {
      color: red;
    }

    span.code {
      font-family: "Courier New", courier, monotype;
    }

    span.function {
      font-weight: bold;
      font-family: "Courier New", courier, monotype;
    }

    table.variables {
      border-collapse: collapse;
      margin-left: 1em;
    }

    td.varName {
      vertical-align: top;
      font-weight: bold;
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    td.varValue {
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    div.variables {
      margin-bottom: 0.5em;
    }

    span.heading {
      font-weight: bold;
    }

    div.dict {
      background: #cccc99;
      padding: 2px;
      float: left;
    }

    td.dictKey {
      background: #ffff99;
      font-weight: bold;
    }

    td.dictValue {
      background: #ffff99;
    }

    div.list {
      background: #7777cc;
      padding: 2px;
      float: left;
    }

    div.listItem {
      background: #9999ff;
    }

    div.instance {
      background: #cc7777;
      padding: 2px;
      float: left;
    }

    span.instanceName {
      font-weight: bold;
      display: block;
    }

    span.instanceRepr {
      background: #ff9999;
      font-family: "Courier New", courier, monotype;
    }

    div.function {
      background: orange;
      font-weight: bold;
      float: left;
    }
</style>
<a href="#tbend">
<p class="error">exceptions.IOError: [Errno 28] No space left on device</p>
</a><div class="stackTrace"></div><a name="tbend"> </a>
<p class="error">exceptions.IOError: [Errno 28] No space left on device</p>
Traceback from remote host -- Traceback (most recent call last):
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/spread/pb.py", line 514, in expressionReceived
    method(*sexp[1:])
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/spread/pb.py", line 884, in proto_answer
    d.callback(self.unserialize(netResult))
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/internet/defer.py", line 361, in callback
    self._startRunCallbacks(result)
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/internet/defer.py", line 455, in _startRunCallbacks
    self._runCallbacks()
--- <exception caught here> ---
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/internet/defer.py", line 542, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
  File "/tools/buildbot/lib/python2.7/site-packages/buildslave/commands/transfer.py", line 362, in finished
    self.fp.close()
exceptions.IOError: [Errno 28] No space left on device
========= Finished download exception (results: 4, elapsed: 30 secs) (at 2015-10-23 15:05:05.821853) =========

========= Skipped  (results: not started, elapsed: not started) =========
========= Skipped  (results: not started, elapsed: not started) =========
========= Started set_script_properties exception (results: 4, elapsed: 0 secs) (at 2015-10-23 15:05:05.822191) =========

remoteFailed: [Failure instance: Traceback from remote host -- Traceback (most recent call last):
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/spread/pb.py", line 840, in _recvMessage
    netResult = object.remoteMessageReceived(self, message, netArgs, netKw)
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/spread/flavors.py", line 114, in remoteMessageReceived
    state = method(*args, **kw)
  File "/tools/buildbot/lib/python2.7/site-packages/buildslave/bot.py", line 141, in remote_startCommand
    d = self.command.doStart()
  File "/tools/buildbot/lib/python2.7/site-packages/buildslave/commands/base.py", line 140, in doStart
    d = defer.maybeDeferred(self.start)
--- <exception caught here> ---
  File "/tools/buildbot/lib/python2.7/site-packages/twisted/internet/defer.py", line 133, in maybeDeferred
    result = f(*args, **kw)
  File "/tools/buildbot/lib/python2.7/site-packages/buildslave/commands/shell.py", line 77, in start
    logEnviron=args.get('logEnviron', True),
  File "/tools/buildbot/lib/python2.7/site-packages/buildslave/runprocess.py", line 281, in __init__
    os.makedirs(workdir)
  File "/tools/buildbot/lib/python2.7/os.py", line 157, in makedirs
    mkdir(name, mode)
exceptions.OSError: [Errno 28] No space left on device: '/builds/slave/fx-team-l64-asan-d-00000000000/properties'
]

<style type="text/css">
    p.error {
      color: red;
      font-family: Verdana, Arial, helvetica, sans-serif;
      font-weight: bold;
    }

    div {
      font-family: Verdana, Arial, helvetica, sans-serif;
    }

    div.stackTrace {
    }

    div.frame {
      padding: 1em;
      background: white;
      border-bottom: thin black dashed;
    }

    div.firstFrame {
      padding: 1em;
      background: white;
      border-top: thin black dashed;
      border-bottom: thin black dashed;
    }

    div.location {
    }

    div.snippet {
      margin-bottom: 0.5em;
      margin-left: 1em;
      background: #FFFFDD;
    }

    div.snippetHighlightLine {
      color: red;
    }

    span.code {
      font-family: "Courier New", courier, monotype;
    }

    span.function {
      font-weight: bold;
      font-family: "Courier New", courier, monotype;
    }

    table.variables {
      border-collapse: collapse;
      margin-left: 1em;
    }

    td.varName {
      vertical-align: top;
      font-weight: bold;
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    td.varValue {
      padding-left: 0.5em;
      padding-right: 0.5em;
    }

    div.variables {
      margin-bottom: 0.5em;
    }

    span.heading {
      font-weight: bold;
    }

    div.dict {
      background: #cccc99;
      padding: 2px;
      float: left;
    }

    td.dictKey {
      background: #ffff99;
      font-weight: bold;
    }

    td.dictValue {
      background: #ffff99;
    }

    div.list {
      background: #7777cc;
      padding: 2px;
      float: left;
    }

    div.listItem {
      background: #9999ff;
    }

    div.instance {
      background: #cc7777;
      padding: 2px;
      float: left;
    }

    span.instanceName {
      font-weight: bold;
      display: block;
    }

    span.instanceRepr {
      background: #ff9999;
      font-family: "Courier New", courier, monotype;
    }

    div.function {
      background: orange;
      font-weight: bold;
      float: left;
    }
</style>
<a href="#tbend">
<p class="error">exceptions.OSError: [Errno 28] No space left on device: '/builds/slave/fx-team-l64-asan-d-00000000000/properties'</p>
</a><div class="stackTrace"></div><a name="tbend"> </a>
<p class="error">exceptions.OSError: [Errno 28] No space left on device: '/builds/slave/fx-team-l64-asan-d-00000000000/properties'</p>
Traceback (most recent call last):
Failure: exceptions.OSError
========= Finished set_script_properties exception (results: 4, elapsed: 0 secs) (at 2015-10-23 15:05:05.979418) =========

========= Started 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:05:05.979758) =========
rm -f oauth.txt
 in dir /builds/slave/fx-team-l64-asan-d-00000000000/. (timeout 1200 secs)
 watching logfiles {}
 argv: ['rm', '-f', 'oauth.txt']
 environment:
  CCACHE_HASHDIR=
  G_BROKEN_FILENAMES=1
  HISTCONTROL=ignoredups
  HISTSIZE=1000
  HOME=/home/cltbld
  HOSTNAME=bld-linux64-spot-551.build.releng.usw2.mozilla.com
  LANG=en_US.UTF-8
  LESSOPEN=|/usr/bin/lesspipe.sh %s
  LOGNAME=cltbld
  MAIL=/var/spool/mail/cltbld
  PATH=/usr/local/bin:/usr/lib64/ccache:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/home/cltbld/bin
  PWD=/builds/slave/fx-team-l64-asan-d-00000000000
  SHELL=/bin/bash
  SHLVL=1
  TERM=linux
  TMOUT=86400
  USER=cltbld
  _=/tools/buildbot/bin/python
 using PTY: False
program finished with exit code 0
elapsedTime=0.003121
========= Finished 'rm -f ...' (results: 0, elapsed: 0 secs) (at 2015-10-23 15:05:06.112428) =========

========= Started reboot slave lost (results: 0, elapsed: 2 mins, 47 secs) (at 2015-10-23 15:05:06.112737) =========

========= Finished reboot slave lost (results: 0, elapsed: 2 mins, 47 secs) (at 2015-10-23 15:07:53.114965) =========