| OLD | NEW |
|---|
| 1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
| 2 # | 2 # |
| 3 # Copyright 2007 Google Inc. | 3 # Copyright 2007 Google Inc. |
| 4 # | 4 # |
| 5 # Licensed under the Apache License, Version 2.0 (the "License"); | 5 # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 # you may not use this file except in compliance with the License. | 6 # you may not use this file except in compliance with the License. |
| 7 # You may obtain a copy of the License at | 7 # You may obtain a copy of the License at |
| 8 # | 8 # |
| 9 # http://www.apache.org/licenses/LICENSE-2.0 | 9 # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 # | 10 # |
| 11 # Unless required by applicable law or agreed to in writing, software | 11 # Unless required by applicable law or agreed to in writing, software |
| 12 # distributed under the License is distributed on an "AS IS" BASIS, | 12 # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 # See the License for the specific language governing permissions and | 14 # See the License for the specific language governing permissions and |
| 15 # limitations under the License. | 15 # limitations under the License. |
| 16 | 16 |
| 17 """Tool for uploading subversion diffs to the codereview app. | 17 """Tool for uploading subversion diffs to the codereview app. |
| 18 | 18 |
| 19 Usage summary: upload.py [options] [-- svn_diff_options] | 19 Usage summary: upload.py [options] [-- svn_diff_options] |
| 20 """ | 20 """ |
| 21 # This code is derived from appcfg.py in the App Engine SDK (open source), | 21 # This code is derived from appcfg.py in the App Engine SDK (open source), |
| 22 # and from ASPN recipe #146306. | 22 # and from ASPN recipe #146306. |
| 23 | 23 |
| 24 import cookielib | 24 import cookielib |
| 25 import getpass | 25 import getpass |
| 26 import logging | 26 import logging |
| 27 import mimetypes | 27 import mimetypes |
| 28 import optparse | 28 import optparse |
| 29 import os | 29 import os |
| 30 import socket | 30 import socket |
| 31 import sys | 31 import sys |
| 32 import urllib | 32 import urllib |
| 33 import urllib2 | 33 import urllib2 |
| 34 import urlparse | 34 import urlparse |
| 35 | 35 |
| 36 | 36 |
| 37 # The logging verbosity: | 37 # The logging verbosity: |
| 38 # 0: Errors only. | 38 # 0: Errors only. |
| 39 # 1: Status messages. | 39 # 1: Status messages. |
| 40 # 2: Info logs. | 40 # 2: Info logs. |
| 41 # 3: Debug logs. | 41 # 3: Debug logs. |
| 42 verbosity = 1 | 42 verbosity = 1 |
| 43 | 43 |
| 44 | 44 |
| 45 def StatusUpdate(msg): | 45 def StatusUpdate(msg): |
| 46 """Print a status message to stdout. | 46 """Print a status message to stdout. |
| 47 | 47 |
| 48 If 'verbosity' is greater than 0, print the message. | 48 If 'verbosity' is greater than 0, print the message. |
| 49 | 49 |
| 50 Args: | 50 Args: |
| (...skipping 263 matching lines...) Show 10 above Show 10 below |
| 314 if os.path.exists(self.cookie_file): | 314 if os.path.exists(self.cookie_file): |
| 315 try: | 315 try: |
| 316 self.cookie_jar.load() | 316 self.cookie_jar.load() |
| 317 self.authenticated = True | 317 self.authenticated = True |
| 318 StatusUpdate("Loaded authentication cookies from %s" % | 318 StatusUpdate("Loaded authentication cookies from %s" % |
| 319 self.cookie_file) | 319 self.cookie_file) |
| 320 except cookielib.LoadError: | 320 except cookielib.LoadError: |
| 321 # Failed to load cookies - just ignore them. | 321 # Failed to load cookies - just ignore them. |
| 322 pass | 322 pass |
| 323 else: | 323 else: |
| 324 # Create an empty cookie file with mode 600 | 324 # Create an empty cookie file with mode 600 |
| 325 fd = os.open(self.cookie_file, os.O_CREAT, 0600) | 325 fd = os.open(self.cookie_file, os.O_CREAT, 0600) |
| 326 os.close(fd) | 326 os.close(fd) |
| 327 # Always chmod the cookie file | 327 # Always chmod the cookie file |
| 328 os.chmod(self.cookie_file, 0600) | 328 os.chmod(self.cookie_file, 0600) |
| 329 else: | 329 else: |
| 330 # Don't save cookies across runs of update.py. | 330 # Don't save cookies across runs of update.py. |
| 331 self.cookie_jar = cookielib.CookieJar() | 331 self.cookie_jar = cookielib.CookieJar() |
| 332 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) | 332 opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) |
| 333 return opener | 333 return opener |
| 334 | 334 |
| 335 | 335 |
| 336 parser = optparse.OptionParser(usage="%prog [options] [-- svn_diff_options]") | 336 parser = optparse.OptionParser(usage="%prog [options] [-- svn_diff_options]") |
| 337 parser.add_option("-q", "--quiet", action="store_const", const=0, | 337 parser.add_option("-q", "--quiet", action="store_const", const=0, |
| 338 dest="verbose", help="Print errors only.") | 338 dest="verbose", help="Print errors only.") |
| 339 parser.add_option("-v", "--verbose", action="store_const", const=2, | 339 parser.add_option("-v", "--verbose", action="store_const", const=2, |
| 340 dest="verbose", default=1, | 340 dest="verbose", default=1, |
| 341 help="Print info level logs.") | 341 help="Print info level logs.") |
| 342 parser.add_option("--noisy", action="store_const", const=3, | 342 parser.add_option("--noisy", action="store_const", const=3, |
| 343 dest="verbose", help="Print all logs.") | 343 dest="verbose", help="Print all logs.") |
| 344 parser.add_option("-s", "--server", action="store", dest="server", | 344 parser.add_option("-s", "--server", action="store", dest="server", |
| 345 default="codereview.appspot.com", | 345 default="codereview.appspot.com", |
| 346 metavar="SERVER", | 346 metavar="SERVER", |
| 347 help="The server to upload to. The format is host[:port].") | 347 help="The server to upload to. The format is host[:port].") |
| 348 parser.add_option("-e", "--email", action="store", dest="email", | 348 parser.add_option("-e", "--email", action="store", dest="email", |
| 349 metavar="EMAIL", default=None, | 349 metavar="EMAIL", default=None, |
| 350 help="The username to use. Will prompt if omitted.") | 350 help="The username to use. Will prompt if omitted.") |
| 351 parser.add_option("-H", "--host", action="store", dest="host", | 351 parser.add_option("-H", "--host", action="store", dest="host", |
| 352 metavar="HOST", default=None, | 352 metavar="HOST", default=None, |
| 353 help="Overrides the Host header sent with all RPCs.") | 353 help="Overrides the Host header sent with all RPCs.") |
| 354 parser.add_option("--no_cookies", action="store_false", | 354 parser.add_option("--no_cookies", action="store_false", |
| 355 dest="save_cookies", default=True, | 355 dest="save_cookies", default=True, |
| 356 help="Do not save authentication cookies to local disk.") | 356 help="Do not save authentication cookies to local disk.") |
| 357 parser.add_option("-m", "--message", action="store", dest="message", | 357 parser.add_option("-m", "--message", action="store", dest="message", |
| 358 metavar="MESSAGE", default=None, | 358 metavar="MESSAGE", default=None, |
| 359 help="A message to identify the patch. " | 359 help="A message to identify the patch. " |
| 360 "Will prompt if omitted.") | 360 "Will prompt if omitted.") |
| 361 parser.add_option("-i", "--issue", type="int", action="store", | 361 parser.add_option("-i", "--issue", type="int", action="store", |
| 362 metavar="ISSUE", default=None, | 362 metavar="ISSUE", default=None, |
| 363 help="Issue number to which to add. Defaults to new issue.") | 363 help="Issue number to which to add. Defaults to new issue.") |
| 364 parser.add_option("-l", "--local_base", action="store_true", |
| 365 dest="local_base", default=False, |
| 366 help="base file will be uploaded") |
| 364 | 367 |
| 365 | 368 |
| 366 def GetRpcServer(options): | 369 def GetRpcServer(options): |
| 367 """Returns an instance of an AbstractRpcServer. | 370 """Returns an instance of an AbstractRpcServer. |
| 368 | 371 |
| 369 Returns: | 372 Returns: |
| 370 A new AbstractRpcServer, on which RPC calls can be made. | 373 A new AbstractRpcServer, on which RPC calls can be made. |
| 371 """ | 374 """ |
| 372 | 375 |
| 373 rpc_server_class = HttpRpcServer | 376 rpc_server_class = HttpRpcServer |
| 374 | 377 |
| 375 def GetUserCredentials(): | 378 def GetUserCredentials(): |
| 376 """Prompts the user for a username and password.""" | 379 """Prompts the user for a username and password.""" |
| 377 email = options.email | 380 email = options.email |
| 378 if email is None: | 381 if email is None: |
| 379 email = raw_input("Email: ").strip() | 382 email = raw_input("Email: ").strip() |
| 380 password = getpass.getpass("Password for %s: " % email) | 383 password = getpass.getpass("Password for %s: " % email) |
| 381 return (email, password) | 384 return (email, password) |
| 382 | 385 |
| 383 # If this is the dev_appserver, use fake authentication. | 386 # If this is the dev_appserver, use fake authentication. |
| 384 host = (options.host or options.server).lower() | 387 host = (options.host or options.server).lower() |
| 385 if host == "localhost" or host.startswith("localhost:"): | 388 if host == "localhost" or host.startswith("localhost:"): |
| 386 email = options.email | 389 email = options.email |
| 387 if email is None: | 390 if email is None: |
| 388 email = "test@example.com" | 391 email = "test@example.com" |
| 389 logging.info("Using debug user %s. Override with --email" % email) | 392 logging.info("Using debug user %s. Override with --email" % email) |
| 390 server = rpc_server_class( | 393 server = rpc_server_class( |
| 391 options.server, | 394 options.server, |
| 392 lambda: (email, "password"), | 395 lambda: (email, "password"), |
| 393 host_override=options.host, | 396 host_override=options.host, |
| 394 extra_headers={"Cookie": | 397 extra_headers={"Cookie": |
| 395 'dev_appserver_login="%s:False"' % email}, | 398 'dev_appserver_login="%s:False"' % email}, |
| 396 save_cookies=options.save_cookies) | 399 save_cookies=options.save_cookies) |
| 397 # Don't try to talk to ClientLogin. | 400 # Don't try to talk to ClientLogin. |
| 398 server.authenticated = True | 401 server.authenticated = True |
| 399 return server | 402 return server |
| 400 | 403 |
| 401 return rpc_server_class(options.server, GetUserCredentials, | 404 return rpc_server_class(options.server, GetUserCredentials, |
| 402 host_override=options.host, | 405 host_override=options.host, |
| 403 save_cookies=options.save_cookies) | 406 save_cookies=options.save_cookies) |
| 404 | 407 |
| 405 | 408 |
| 406 def EncodeMultipartFormData(fields, files): | 409 def EncodeMultipartFormData(fields, files): |
| 407 """Encode form fields for multipart/form-data. | 410 """Encode form fields for multipart/form-data. |
| 408 | 411 |
| 409 Args: | 412 Args: |
| 410 fields: A sequence of (name, value) elements for regular form fields. | 413 fields: A sequence of (name, value) elements for regular form fields. |
| 411 files: A sequence of (name, filename, value) elements for data to be | 414 files: A sequence of (name, filename, value) elements for data to be |
| 412 uploaded as files. | 415 uploaded as files. |
| 413 Returns: | 416 Returns: |
| 414 (content_type, body) ready for httplib.HTTP instance. | 417 (content_type, body) ready for httplib.HTTP instance. |
| 415 | 418 |
| 416 Source: | 419 Source: |
| 417 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 | 420 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/146306 |
| 418 """ | 421 """ |
| 419 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' | 422 BOUNDARY = '-M-A-G-I-C---B-O-U-N-D-A-R-Y-' |
| 420 CRLF = '\r\n' | 423 CRLF = '\r\n' |
| 421 lines = [] | 424 lines = [] |
| 422 for (key, value) in fields: | 425 for (key, value) in fields: |
| 423 lines.append('--' + BOUNDARY) | 426 lines.append('--' + BOUNDARY) |
| 424 lines.append('Content-Disposition: form-data; name="%s"' % key) | 427 lines.append('Content-Disposition: form-data; name="%s"' % key) |
| 425 lines.append('') | 428 lines.append('') |
| 426 lines.append(value) | 429 lines.append(value) |
| 427 for (key, filename, value) in files: | 430 for (key, filename, value) in files: |
| 428 lines.append('--' + BOUNDARY) | 431 lines.append('--' + BOUNDARY) |
| 429 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % | 432 lines.append('Content-Disposition: form-data; name="%s"; filename="%s"' % |
| 430 (key, filename)) | 433 (key, filename)) |
| 431 lines.append('Content-Type: %s' % GetContentType(filename)) | 434 lines.append('Content-Type: %s' % GetContentType(filename)) |
| 432 lines.append('') | 435 lines.append('') |
| 433 lines.append(value) | 436 lines.append(value) |
| 434 lines.append('--' + BOUNDARY + '--') | 437 lines.append('--' + BOUNDARY + '--') |
| 435 lines.append('') | 438 lines.append('') |
| 436 body = CRLF.join(lines) | 439 body = CRLF.join(lines) |
| 437 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY | 440 content_type = 'multipart/form-data; boundary=%s' % BOUNDARY |
| 438 return content_type, body | 441 return content_type, body |
| 439 | 442 |
| 440 | 443 |
| 441 def GetContentType(filename): | 444 def GetContentType(filename): |
| 442 """Helper to guess the content-type from the filename.""" | 445 """Helper to guess the content-type from the filename.""" |
| 443 return mimetypes.guess_type(filename)[0] or 'application/octet-stream' | 446 return mimetypes.guess_type(filename)[0] or 'application/octet-stream' |
| 444 | 447 |
| 445 | 448 |
| 446 def RunShell(command, args=(), silent_ok=False): | 449 def RunShell(command, args=(), silent_ok=False): |
| 447 logging.info("Running %s", command) | 450 logging.info("Running %s", command) |
| 448 stream = os.popen("%s %s" % (command, " ".join(args)), "r") | 451 stream = os.popen("%s %s" % (command, " ".join(args)), "r") |
| 449 data = stream.read() | 452 data = stream.read() |
| 450 if stream.close(): | 453 if stream.close(): |
| 451 ErrorExit("Got error status from %s" % command) | 454 ErrorExit("Got error status from %s" % command) |
| 452 if not silent_ok and not data: | 455 if not silent_ok and not data: |
| 453 ErrorExit("No output from %s" % command) | 456 ErrorExit("No output from %s" % command) |
| 454 return data | 457 return data |
| 455 | 458 |
| 456 | 459 |
| 457 def GuessBase(): | 460 def GuessBase(): |
| 458 info = RunShell("svn info") | 461 info = RunShell("svn info") |
| 459 for line in info.splitlines(): | 462 for line in info.splitlines(): |
| 460 words = line.split() | 463 words = line.split() |
| 461 if len(words) == 2 and words[0] == "URL:": | 464 if len(words) == 2 and words[0] == "URL:": |
| 462 url = words[1] | 465 url = words[1] |
| 463 scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) | 466 scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) |
| 464 if netloc.endswith("svn.python.org"): | 467 if netloc.endswith("svn.python.org"): |
| 465 if netloc == "svn.python.org": | 468 if netloc == "svn.python.org": |
| 466 if path.startswith("/projects/"): | 469 if path.startswith("/projects/"): |
| 467 path = path[9:] | 470 path = path[9:] |
| 468 elif netloc != "pythondev@svn.python.org": | 471 elif netloc != "pythondev@svn.python.org": |
| 469 ErrorExit("Unrecognized Python URL: %s" % url) | 472 ErrorExit("Unrecognized Python URL: %s" % url) |
| 470 base = "http://svn.python.org/view/*checkout*%s/" % path | 473 base = "http://svn.python.org/view/*checkout*%s/" % path |
| 471 logging.info("Guessed Python base = %s", base) | 474 logging.info("Guessed Python base = %s", base) |
| 472 elif netloc.endswith("svn.collab.net"): | 475 elif netloc.endswith("svn.collab.net"): |
| 473 if path.startswith("/repos/"): | 476 if path.startswith("/repos/"): |
| 474 path = path[6:] | 477 path = path[6:] |
| 475 base = "http://svn.collab.net/viewvc/*checkout*%s/" % path | 478 base = "http://svn.collab.net/viewvc/*checkout*%s/" % path |
| 476 logging.info("Guessed CollabNet base = %s", base) | 479 logging.info("Guessed CollabNet base = %s", base) |
| 477 elif netloc.endswith(".googlecode.com"): | 480 elif netloc.endswith(".googlecode.com"): |
| 478 base = url + "/" | 481 base = url + "/" |
| 479 if base.startswith("https"): | 482 if base.startswith("https"): |
| 480 base = "http" + base[5:] | 483 base = "http" + base[5:] |
| 481 logging.info("Guessed Google Code base = %s", base) | 484 logging.info("Guessed Google Code base = %s", base) |
| 482 else: | 485 else: |
| 483 ErrorExit("Unrecognized svn project root: %s" % url) | 486 ErrorExit("Unrecognized svn project root: %s" % url) |
| 484 return base | 487 return base |
| 485 ErrorExit("Can't find URL in output from svn info") | 488 ErrorExit("Can't find URL in output from svn info") |
| 486 | 489 |
| 487 | 490 |
| 488 def RealMain(argv): | 491 def RealMain(argv): |
| 489 logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:" | 492 logging.basicConfig(format=("%(asctime).19s %(levelname)s %(filename)s:" |
| 490 "%(lineno)s %(message)s ")) | 493 "%(lineno)s %(message)s ")) |
| 491 options, args = parser.parse_args(sys.argv[1:]) | 494 options, args = parser.parse_args(sys.argv[1:]) |
| 492 global verbosity | 495 global verbosity |
| 493 verbosity = options.verbose | 496 verbosity = options.verbose |
| 494 if verbosity >= 3: | 497 if verbosity >= 3: |
| 495 logging.getLogger().setLevel(logging.DEBUG) | 498 logging.getLogger().setLevel(logging.DEBUG) |
| 496 elif verbosity >= 2: | 499 elif verbosity >= 2: |
| 497 logging.getLogger().setLevel(logging.INFO) | 500 logging.getLogger().setLevel(logging.INFO) |
| 498 base = GuessBase() | 501 if options.local_base: |
| 502 base = None |
| 503 else: |
| 504 base = GuessBase() |
| 499 CheckForUnknownFiles() | 505 CheckForUnknownFiles() |
| 500 data = RunShell("svn diff", args) | 506 data = RunShell("svn diff", args) |
| 501 count = 0 | 507 count = 0 |
| 502 for line in data.splitlines(): | 508 for line in data.splitlines(): |
| 503 if line.startswith("Index:"): | 509 if line.startswith("Index:"): |
| 504 count += 1 | 510 count += 1 |
| 505 logging.info(line) | 511 logging.info(line) |
| 506 if not count: | 512 if not count: |
| 507 ErrorExit("No valid patches found in output from svn diff") | 513 ErrorExit("No valid patches found in output from svn diff") |
| 508 if options.issue: | 514 if options.issue: |
| 509 prompt = "Message describing this patch set: " | 515 prompt = "Message describing this patch set: " |
| 510 else: | 516 else: |
| 511 prompt = "New issue subject: " | 517 prompt = "New issue subject: " |
| 512 message = options.message or raw_input(prompt).strip() | 518 message = options.message or raw_input(prompt).strip() |
| 513 if not message: | 519 if not message: |
| 514 ErrorExit("A non-empty message is required") | 520 ErrorExit("A non-empty message is required") |
| 515 rpc_server = GetRpcServer(options) | 521 rpc_server = GetRpcServer(options) |
| 516 form_fields = [("base", base), ("subject", message)] | 522 form_fields = [("subject", message)] |
| 523 if base is not None: |
| 524 form_fields.append(("base", base)) |
| 517 if options.issue: | 525 if options.issue: |
| 518 form_fields.append(("issue", str(options.issue))) | 526 form_fields.append(("issue", str(options.issue))) |
| 519 if options.email: | 527 if options.email: |
| 520 form_fields.append(("user", options.email)) | 528 form_fields.append(("user", options.email)) |
| 521 ctype, body = EncodeMultipartFormData(form_fields, | 529 ctype, body = EncodeMultipartFormData(form_fields, |
| 522 [("data", "data.diff", data)]) | 530 [("data", "data.diff", data)]) |
| 523 response_body = rpc_server.Send("/upload", body, content_type=ctype) | 531 response_body = rpc_server.Send("/upload", body, content_type=ctype) |
| 524 StatusUpdate(response_body) | 532 StatusUpdate(response_body) |
| 525 sys.exit(not response_body.startswith("Issue created.")) | 533 sys.exit(not response_body.startswith("Issue created.")) |
| 526 | 534 |
| 527 def CheckForUnknownFiles(): | 535 def CheckForUnknownFiles(): |
| 528 status = RunShell("svn status --ignore-externals", silent_ok=True) | 536 status = RunShell("svn status --ignore-externals", silent_ok=True) |
| 529 unknown_files = [] | 537 unknown_files = [] |
| 530 for line in status.split("\n"): | 538 for line in status.split("\n"): |
| 531 if line and line[0] == "?": | 539 if line and line[0] == "?": |
| 532 unknown_files.append(line) | 540 unknown_files.append(line) |
| 533 if unknown_files: | 541 if unknown_files: |
| 534 print "The following files are not added to version control:" | 542 print "The following files are not added to version control:" |
| 535 for line in unknown_files: | 543 for line in unknown_files: |
| 536 print line | 544 print line |
| 537 prompt = "Are you sure to continue?(y/N) " | 545 prompt = "Are you sure to continue?(y/N) " |
| 538 answer = raw_input(prompt).strip() | 546 answer = raw_input(prompt).strip() |
| 539 if answer != "y": | 547 if answer != "y": |
| 540 ErrorExit("User aborted") | 548 ErrorExit("User aborted") |
| 541 | 549 |
| 542 def main(): | 550 def main(): |
| 543 try: | 551 try: |
| 544 RealMain(sys.argv) | 552 RealMain(sys.argv) |
| 545 except KeyboardInterrupt: | 553 except KeyboardInterrupt: |
| 546 print | 554 print |
| 547 StatusUpdate("Interrupted.") | 555 StatusUpdate("Interrupted.") |
| 548 sys.exit(1) | 556 sys.exit(1) |
| 549 | 557 |
| 550 | 558 |
| 551 if __name__ == "__main__": | 559 if __name__ == "__main__": |
| 552 main() | 560 main() |
| OLD | NEW |
|---|
