# DistUpgradeController.py # # Copyright (c) 2004-2008 Canonical # # Author: Michael Vogt # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as # published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA from __future__ import absolute_import, print_function import warnings warnings.filterwarnings("ignore", "apt API not stable yet", FutureWarning) import apt import apt_pkg import sys import os import subprocess import locale import logging import shutil import glob import time import copy try: # >= 3.0 from configparser import NoOptionError if sys.version >= '3.2': from configparser import ConfigParser as SafeConfigParser else: from configparser import SafeConfigParser except ImportError: # < 3.0 from ConfigParser import SafeConfigParser, NoOptionError from .utils import (country_mirror, url_downloadable, check_and_fix_xbit, get_arch, iptables_active, inside_chroot, get_string_with_no_auth_from_source_entry, is_child_of_process_name) from string import Template try: from urllib.parse import urlsplit except ImportError: from urlparse import urlsplit from .DistUpgradeView import STEP_PREPARE, STEP_MODIFY_SOURCES, STEP_FETCH, STEP_INSTALL, STEP_CLEANUP, STEP_REBOOT from .DistUpgradeCache import MyCache from .DistUpgradeConfigParser import DistUpgradeConfig from .DistUpgradeQuirks import DistUpgradeQuirks from .DistUpgradeAptCdrom import AptCdrom from .DistUpgradeAufs import setupAufs, aufsOptionsAndEnvironmentSetup # workaround broken relative import in python-apt (LP: #871007), we # want the local version of distinfo.py from oneiric, but because of # a bug in python-apt we will get the natty version that does not # know about "Component.parent_component" leading to a crash from . import distinfo from . import sourceslist sourceslist.DistInfo = distinfo.DistInfo from .sourceslist import SourcesList, is_mirror from .distro import get_distro, NoDistroTemplateException from .DistUpgradeGettext import gettext as _ from .DistUpgradeGettext import ngettext import gettext from .DistUpgradeCache import (CacheExceptionDpkgInterrupted, CacheExceptionLockingFailed, NotEnoughFreeSpaceError) from .DistUpgradeApport import run_apport REBOOT_REQUIRED_FILE = "/var/run/reboot-required" def component_ordering_key(a): """ key() function for sorted to ensure "correct" component ordering """ ordering = ["main", "restricted", "universe", "multiverse"] try: return ordering.index(a) except ValueError: # ensure to sort behind the "official" components, order is not # really important for those return len(ordering)+1 class NoBackportsFoundException(Exception): pass class DistUpgradeController(object): """ this is the controller that does most of the work """ def __init__(self, distUpgradeView, options=None, datadir=None): # setup the paths localedir = "/usr/share/locale/" if datadir == None or datadir == '.': datadir = os.getcwd() localedir = os.path.join(datadir,"mo") self.datadir = datadir self.options = options # init gettext gettext.bindtextdomain("ubuntu-release-upgrader",localedir) gettext.textdomain("ubuntu-release-upgrader") # setup the view logging.debug("Using '%s' view" % distUpgradeView.__class__.__name__) self._view = distUpgradeView self._view.updateStatus(_("Reading cache")) self.cache = None if not self.options or self.options.withNetwork == None: self.useNetwork = True else: self.useNetwork = self.options.withNetwork if options: cdrompath = options.cdromPath else: cdrompath = None self.aptcdrom = AptCdrom(distUpgradeView, cdrompath) # the configuration self.config = DistUpgradeConfig(datadir) self.sources_backup_ext = "."+self.config.get("Files","BackupExt") # move some of the options stuff into the self.config, # ConfigParser deals only with strings it seems *sigh* self.config.add_section("Options") self.config.set("Options","withNetwork", str(self.useNetwork)) # aufs stuff aufsOptionsAndEnvironmentSetup(self.options, self.config) # some constants here self.fromDist = self.config.get("Sources","From") self.toDist = self.config.get("Sources","To") self.origin = self.config.get("Sources","ValidOrigin") self.arch = get_arch() # we run with --force-overwrite by default if "RELEASE_UPGRADE_NO_FORCE_OVERWRITE" not in os.environ: logging.debug("enable dpkg --force-overwrite") apt_pkg.config.set("DPkg::Options::","--force-overwrite") # we run in full upgrade mode by default self._partialUpgrade = False # install the quirks handler self.quirks = DistUpgradeQuirks(self, self.config) # setup env var os.environ["RELEASE_UPGRADE_IN_PROGRESS"] = "1" os.environ["PYCENTRAL_FORCE_OVERWRITE"] = "1" os.environ["PATH"] = "%s:%s" % (os.getcwd()+"/imported", os.environ["PATH"]) check_and_fix_xbit("./imported/invoke-rc.d") # set max retries maxRetries = self.config.getint("Network","MaxRetries") apt_pkg.config.set("Acquire::Retries", str(maxRetries)) # max sizes for dpkgpm for large installs (see linux/limits.h and # linux/binfmts.h) apt_pkg.config.set("Dpkg::MaxArgs", str(64*1024)) apt_pkg.config.set("Dpkg::MaxArgBytes", str(128*1024)) # smaller to avoid hangs apt_pkg.config.set("Acquire::http::Timeout","20") apt_pkg.config.set("Acquire::ftp::Timeout","20") # no list cleanup here otherwise a "cancel" in the upgrade # will not restore the full state (lists will be missing) apt_pkg.config.set("Apt::Get::List-Cleanup", "false") # forced obsoletes self.forced_obsoletes = self.config.getlist("Distro","ForcedObsoletes") # list of valid mirrors that we can add self.valid_mirrors = self.config.getListFromFile("Sources","ValidMirrors") # third party mirrors self.valid_3p_mirrors = [] if self.config.has_section('ThirdPartyMirrors'): self.valid_3p_mirrors = [pair[1] for pair in self.config.items('ThirdPartyMirrors')] # debugging #apt_pkg.config.set("DPkg::Options::","--debug=0077") # apt cron job self._aptCronJobPerms = 0o755 def openCache(self, lock=True, restore_sources_list_on_fail=False): logging.debug("openCache()") if self.cache is None: self.quirks.run("PreCacheOpen") else: self.cache.release_lock() self.cache.unlock_lists_dir() # this loop will try getting the lock a couple of times MAX_LOCK_RETRIES = 20 lock_retry = 0 while True: try: # exit here once the cache is ready return self._openCache(lock) except CacheExceptionLockingFailed as e: # wait a bit lock_retry += 1 self._view.processEvents() time.sleep(0.1) logging.debug( "failed to lock the cache, retrying (%i)" % lock_retry) # and give up after some time if lock_retry > MAX_LOCK_RETRIES: logging.error("Cache can not be locked (%s)" % e) self._view.error(_("Unable to get exclusive lock"), _("This usually means that another " "package management application " "(like apt-get or aptitude) " "already running. Please close that " "application first.")); if restore_sources_list_on_fail: self.abort() else: sys.exit(1) def _openCache(self, lock): try: self.cache = MyCache(self.config, self._view, self.quirks, self._view.getOpCacheProgress(), lock) # alias name for the plugin interface code self.apt_cache = self.cache # if we get a dpkg error that it was interrupted, just # run dpkg --configure -a except CacheExceptionDpkgInterrupted: logging.warning("dpkg interrupted, calling dpkg --configure -a") cmd = ["/usr/bin/dpkg","--configure","-a"] if os.environ.get("DEBIAN_FRONTEND") == "noninteractive": cmd.append("--force-confold") self._view.getTerminal().call(cmd) self.cache = MyCache(self.config, self._view, self.quirks, self._view.getOpCacheProgress()) self.cache.partialUpgrade = self._partialUpgrade logging.debug("/openCache(), new cache size %i" % len(self.cache)) def _viewSupportsSSH(self): """ Returns True if this view support upgrades over ssh. In theory all views should support it, but for savety we do only allow text ssh upgrades (see LP: #322482) """ supported = self.config.getlist("View","SupportSSH") if self._view.__class__.__name__ in supported: return True return False def _sshMagic(self): """ this will check for server mode and if we run over ssh. if this is the case, we will ask and spawn a additional daemon (to be sure we have a spare one around in case of trouble) """ pidfile = os.path.join("/var/run/release-upgrader-sshd.pid") if (not os.path.exists(pidfile) and os.path.isdir("/proc") and is_child_of_process_name("sshd")): # check if the frontend supports ssh upgrades (see lp: #322482) if not self._viewSupportsSSH(): logging.error("upgrade over ssh not allowed") self._view.error(_("Upgrading over remote connection not supported"), _("You are running the upgrade over a " "remote ssh connection with a frontend " "that does " "not support this. Please try a text " "mode upgrade with 'do-release-upgrade'." "\n\n" "The upgrade will " "abort now. Please try without ssh.") ) sys.exit(1) return False # ask for a spare one to start (and below 1024) port = 1022 res = self._view.askYesNoQuestion( _("Continue running under SSH?"), _("This session appears to be running under ssh. " "It is not recommended to perform a upgrade " "over ssh currently because in case of failure " "it is harder to recover.\n\n" "If you continue, an additional ssh daemon will be " "started at port '%s'.\n" "Do you want to continue?") % port) # abort if res == False: sys.exit(1) res = subprocess.call(["/usr/sbin/sshd", "-o", "PidFile=%s" % pidfile, "-p",str(port)]) if res == 0: summary = _("Starting additional sshd") descr = _("To make recovery in case of failure easier, an " "additional sshd will be started on port '%s'. " "If anything goes wrong with the running ssh " "you can still connect to the additional one.\n" ) % port if iptables_active(): cmd = "iptables -I INPUT -p tcp --dport %s -j ACCEPT" % port descr += _( "If you run a firewall, you may need to " "temporarily open this port. As this is " "potentially dangerous it's not done automatically. " "You can open the port with e.g.:\n'%s'") % cmd self._view.information(summary, descr) return True def _tryUpdateSelf(self): """ this is a helper that is run if we are started from a CD and we have network - we will then try to fetch a update of ourself """ from .MetaRelease import MetaReleaseCore from .DistUpgradeFetcherSelf import DistUpgradeFetcherSelf # check if we run from a LTS forceLTS=False if (self.release == "dapper" or self.release == "hardy" or self.release == "lucid" or self.release == "precise"): forceLTS=True m = MetaReleaseCore(useDevelopmentRelease=False, forceLTS=forceLTS) # this will timeout eventually self._view.processEvents() while not m.downloaded.wait(0.1): self._view.processEvents() if m.new_dist is None: logging.error("No new dist found") return False # we have a new dist progress = self._view.getAcquireProgress() fetcher = DistUpgradeFetcherSelf(new_dist=m.new_dist, progress=progress, options=self.options, view=self._view) fetcher.run() def _pythonSymlinkCheck(self): """ sanity check that /usr/bin/python points to the default python version. Users tend to modify this symlink, which breaks stuff in obscure ways (Ubuntu #75557). """ logging.debug("_pythonSymlinkCheck run") if os.path.exists('/usr/share/python/debian_defaults'): config = SafeConfigParser() config.readfp(open('/usr/share/python/debian_defaults')) try: expected_default = config.get('DEFAULT', 'default-version') except NoOptionError: logging.debug("no default version for python found in '%s'" % config) return False try: fs_default_version = os.readlink('/usr/bin/python') except OSError as e: logging.error("os.readlink failed (%s)" % e) return False if not fs_default_version in (expected_default, os.path.join('/usr/bin', expected_default)): logging.debug("python symlink points to: '%s', but expected is '%s' or '%s'" % (fs_default_version, expected_default, os.path.join('/usr/bin', expected_default))) return False return True def prepare(self): """ initial cache opening, sanity checking, network checking """ # first check if that is a good upgrade self.release = release = subprocess.Popen(["lsb_release","-c","-s"], stdout=subprocess.PIPE, universal_newlines=True).communicate()[0].strip() logging.debug("lsb-release: '%s'" % release) if not (release == self.fromDist or release == self.toDist): logging.error("Bad upgrade: '%s' != '%s' " % (release, self.fromDist)) self._view.error(_("Can not upgrade"), _("An upgrade from '%s' to '%s' is not " "supported with this tool." % (release, self.toDist))) sys.exit(1) # setup aufs if self.config.getWithDefault("Aufs", "EnableFullOverlay", False): aufs_rw_dir = self.config.get("Aufs","RWDir") if not setupAufs(aufs_rw_dir): logging.error("aufs setup failed") self._view.error(_("Sandbox setup failed"), _("It was not possible to create the sandbox " "environment.")) return False # all good, tell the user about the sandbox mode logging.info("running in aufs overlay mode") self._view.information(_("Sandbox mode"), _("This upgrade is running in sandbox " "(test) mode. All changes are written " "to '%s' and will be lost on the next " "reboot.\n\n" "*No* changes written to a system directory " "from now until the next reboot are " "permanent.") % aufs_rw_dir) # setup backports (if we have them) if self.options and self.options.havePrerequists: backportsdir = os.getcwd()+"/backports" logging.info("using backports in '%s' " % backportsdir) logging.debug("have: %s" % glob.glob(backportsdir+"/*.udeb")) if os.path.exists(backportsdir+"/usr/bin/dpkg"): apt_pkg.config.set("Dir::Bin::dpkg",backportsdir+"/usr/bin/dpkg"); if os.path.exists(backportsdir+"/usr/lib/apt/methods"): apt_pkg.config.set("Dir::Bin::methods",backportsdir+"/usr/lib/apt/methods") conf = backportsdir+"/etc/apt/apt.conf.d/01ubuntu" if os.path.exists(conf): logging.debug("adding config '%s'" % conf) apt_pkg.read_config_file(apt_pkg.config, conf) # do the ssh check and warn if we run under ssh self._sshMagic() # check python version if not self._pythonSymlinkCheck(): logging.error("pythonSymlinkCheck() failed, aborting") self._view.error(_("Can not upgrade"), _("Your python install is corrupted. " "Please fix the '/usr/bin/python' symlink.")) sys.exit(1) # open cache try: self.openCache() except SystemError as e: logging.error("openCache() failed: '%s'" % e) return False if not self.cache.sanity_check(self._view): return False # now figure out if we need to go into desktop or # server mode - we use a heuristic for this self.serverMode = self.cache.need_server_mode() if self.serverMode: os.environ["RELEASE_UPGRADE_MODE"] = "server" else: os.environ["RELEASE_UPGRADE_MODE"] = "desktop" if not self.checkViewDepends(): logging.error("checkViewDepends() failed") return False if os.path.exists("/usr/bin/debsig-verify"): logging.error("debsig-verify is installed") self._view.error(_("Package 'debsig-verify' is installed"), _("The upgrade can not continue with that " "package installed.\n" "Please remove it with synaptic " "or 'apt-get remove debsig-verify' first " "and run the upgrade again.")) self.abort() from .DistUpgradeMain import SYSTEM_DIRS for systemdir in SYSTEM_DIRS: if os.path.exists(systemdir) and not os.access(systemdir, os.W_OK): logging.error("%s not writable" % systemdir) self._view.error( _("Can not write to '%s'") % systemdir, _("Its not possible to write to the system directory " "'%s' on your system. The upgrade can not " "continue.\n" "Please make sure that the system directory is " "writable.") % systemdir) self.abort() # FIXME: we may try to find out a bit more about the network # connection here and ask more intelligent questions if self.aptcdrom and self.options and self.options.withNetwork == None: res = self._view.askYesNoQuestion(_("Include latest updates from the Internet?"), _("The upgrade system can use the internet to " "automatically download " "the latest updates and install them during the " "upgrade. If you have a network connection this is " "highly recommended.\n\n" "The upgrade will take longer, but when " "it is complete, your system will be fully up to " "date. You can choose not to do this, but you " "should install the latest updates soon after " "upgrading.\n" "If you answer 'no' here, the network is not " "used at all."), 'Yes') self.useNetwork = res self.config.set("Options","withNetwork", str(self.useNetwork)) logging.debug("useNetwork: '%s' (selected by user)" % res) if res: self._tryUpdateSelf() return True def _sourcesListEntryDownloadable(self, entry): """ helper that checks if a sources.list entry points to something downloadable """ logging.debug("verifySourcesListEntry: %s" % entry) # no way to verify without network if not self.useNetwork: logging.debug("skiping downloadable check (no network)") return True # check if the entry points to something we can download uri = "%s/dists/%s/Release" % (entry.uri, entry.dist) return url_downloadable(uri, logging.debug) def rewriteSourcesList(self, mirror_check=True): logging.debug("rewriteSourcesList()") sync_components = self.config.getlist("Sources","Components") # skip mirror check if special environment is set # (useful for server admins with internal repos) if (self.config.getWithDefault("Sources","AllowThirdParty",False) or "RELEASE_UPRADER_ALLOW_THIRD_PARTY" in os.environ): logging.warning("mirror check skipped, *overriden* via config") mirror_check=False # check if we need to enable main if mirror_check == True and self.useNetwork: # now check if the base-meta pkgs are available in # the archive or only available as "now" # -> if not that means that "main" is missing and we # need to enable it for pkgname in self.config.getlist("Distro","BaseMetaPkgs"): if ((not pkgname in self.cache or not self.cache[pkgname].candidate or len(self.cache[pkgname].candidate.origins) == 0) or (self.cache[pkgname].candidate and len(self.cache[pkgname].candidate.origins) == 1 and self.cache[pkgname].candidate.origins[0].archive == "now") ): logging.debug("BaseMetaPkg '%s' has no candidate.origins" % pkgname) try: distro = get_distro() distro.get_sources(self.sources) distro.enable_component("main") except NoDistroTemplateException: # fallback if everything else does not work, # we replace the sources.list with a single # line to ubuntu-main logging.warning('get_distro().enable_component("main") failed, overwriting sources.list instead as last resort') s = "# auto generated by ubuntu-release-upgrader" s += "deb http://archive.ubuntu.com/ubuntu %s main restricted" % self.toDist s += "deb http://archive.ubuntu.com/ubuntu %s-updates main restricted" % self.toDist s += "deb http://security.ubuntu.com/ubuntu %s-security main restricted" % self.toDist open("/etc/apt/sources.list","w").write(s) break # this must map, i.e. second in "from" must be the second in "to" # (but they can be different, so in theory we could exchange # component names here) pockets = self.config.getlist("Sources","Pockets") fromDists = [self.fromDist] + ["%s-%s" % (self.fromDist, x) for x in pockets] toDists = [self.toDist] + ["%s-%s" % (self.toDist,x) for x in pockets] self.sources_disabled = False # look over the stuff we have foundToDist = False # collect information on what components (main,universe) are enabled for what distro (sub)version # e.g. found_components = { 'hardy':set("main","restricted"), 'hardy-updates':set("main") } self.found_components = {} # special case for ESM for entry in self.sources.list[:]: if (not entry.disabled and entry.uri.endswith("esm.ubuntu.com/ubuntu") and entry.dist == self.fromDist): entry.dist = '%s-updates' % self.fromDist sec_entry = copy.copy(entry) sec_entry.dist = '%s-security' % self.fromDist self.sources.list.append(sec_entry) shutil.copy("ubuntu-esm-v2-keyring.gpg", "/etc/apt/trusted.gpg.d/") for entry in self.sources.list[:]: # ignore invalid records or disabled ones if entry.invalid or entry.disabled: continue # we disable breezy cdrom sources to make sure that demoted # packages are removed if entry.uri.startswith("cdrom:") and entry.dist == self.fromDist: logging.debug("disabled '%s' cdrom entry (dist == fromDist)" % entry) entry.disabled = True continue # check if there is actually a lists file for them available # and disable them if not elif entry.uri.startswith("cdrom:"): # listdir = apt_pkg.config.find_dir("Dir::State::lists") if not os.path.exists("%s/%s%s_%s_%s" % (listdir, apt_pkg.uri_to_filename(entry.uri), "dists", entry.dist, "Release")): logging.warning("disabling cdrom source '%s' because it has no Release file" % entry) entry.disabled = True continue # special case for archive.canonical.com that needs to # be rewritten (for pre-gutsy upgrades) cdist = "%s-commercial" % self.fromDist if (not entry.disabled and entry.uri.startswith("http://archive.canonical.com") and entry.dist == cdist): entry.dist = self.toDist entry.comps = ["partner"] logging.debug("transitioned commercial to '%s' " % entry) continue # special case for landscape.canonical.com because they # don't use a standard archive layout (gutsy->hardy) if (not entry.disabled and entry.uri.startswith("http://landscape.canonical.com/packages/%s" % self.fromDist)): logging.debug("commenting landscape.canonical.com out") entry.disabled = True continue # Disable proposed on upgrade to a development release. if (not entry.disabled and self.options and self.options.devel_release == True and "%s-proposed" % self.fromDist in entry.dist): logging.debug("upgrade to development release, disabling proposed") entry.dist = "%s-proposed" % self.toDist entry.comment += _("Not for humans during development stage of release %s") % self.toDist entry.disabled = True continue # handle upgrades from a EOL release and check if there # is a supported release available if (not entry.disabled and "old-releases.ubuntu.com/" in entry.uri): logging.debug("upgrade from old-releases.ubuntu.com detected") # test country mirror first, then archive.u.c for uri in ["http://%sarchive.ubuntu.com/ubuntu" % country_mirror(), "http://archive.ubuntu.com/ubuntu"]: test_entry = copy.copy(entry) test_entry.uri = uri test_entry.dist = self.toDist if self._sourcesListEntryDownloadable(test_entry): logging.info("transition from old-release.u.c to %s" % uri) entry.uri = uri break logging.debug("examining: '%s'" % get_string_with_no_auth_from_source_entry(entry)) # check if it's a mirror (or official site) validMirror = self.isMirror(entry.uri) thirdPartyMirror = not mirror_check or self.isThirdPartyMirror(entry.uri) if validMirror or thirdPartyMirror: # disabled/security/commercial/extras are special cases # we use validTo/foundToDist to figure out if we have a # main archive mirror in the sources.list or if we # need to add one validTo = True if (entry.disabled or entry.type == "deb-src" or "/security.ubuntu.com" in entry.uri or "%s-security" % self.fromDist in entry.dist or "%s-backports" % self.fromDist in entry.dist or "/archive.canonical.com" in entry.uri or "/extras.ubuntu.com" in entry.uri): validTo = False if entry.dist in toDists: # so the self.sources.list is already set to the new # distro logging.debug("entry '%s' is already set to new dist" % get_string_with_no_auth_from_source_entry(entry)) foundToDist |= validTo elif entry.dist in fromDists: foundToDist |= validTo entry.dist = toDists[fromDists.index(entry.dist)] logging.debug("entry '%s' updated to new dist" % get_string_with_no_auth_from_source_entry(entry)) elif entry.type == 'deb-src': continue elif validMirror: # disable all entries that are official but don't # point to either "to" or "from" dist entry.disabled = True self.sources_disabled = True logging.debug("entry '%s' was disabled (unknown dist)" % get_string_with_no_auth_from_source_entry(entry)) # if we make it to this point, we have a official or third-party mirror # check if the arch is powerpc or sparc and if so, transition # to ports.ubuntu.com (powerpc got demoted in gutsy, sparc # in hardy) if (entry.type == "deb" and not "ports.ubuntu.com" in entry.uri and (self.arch == "powerpc" or self.arch == "sparc")): logging.debug("moving %s source entry to 'ports.ubuntu.com' " % self.arch) entry.uri = "http://ports.ubuntu.com/ubuntu-ports/" # gather what components are enabled and are inconsistent for d in ["%s" % self.toDist, "%s-updates" % self.toDist, "%s-security" % self.toDist]: # create entry if needed, ignore disabled # entries and deb-src self.found_components.setdefault(d,set()) if (not entry.disabled and entry.dist == d and entry.type == "deb"): for comp in entry.comps: # only sync components we know about if not comp in sync_components: continue self.found_components[d].add(comp) else: # disable anything that is not from a official mirror or a whitelisted third party if entry.dist == self.fromDist: entry.dist = self.toDist disable_comment = " " + _("disabled on upgrade to %s") % self.toDist if isinstance(entry.comment, bytes): entry.comment += disable_comment.encode('UTF-8') else: entry.comment += disable_comment entry.disabled = True self.sources_disabled = True logging.debug("entry '%s' was disabled (unknown mirror)" % get_string_with_no_auth_from_source_entry(entry)) # now go over the list again and check for missing components # in $dist-updates and $dist-security and add them for entry in self.sources.list[:]: # skip all comps that are not relevant (including e.g. "hardy") if (entry.invalid or entry.disabled or entry.type == "deb-src" or entry.uri.startswith("cdrom:") or entry.dist == self.toDist): continue # now check for "$dist-updates" and "$dist-security" and add any inconsistencies if entry.dist in self.found_components: component_diff = self.found_components[self.toDist]-self.found_components[entry.dist] if component_diff: logging.info("fixing components inconsistency from '%s'" % get_string_with_no_auth_from_source_entry(entry)) # extend and make sure to keep order entry.comps.extend( sorted(component_diff, key=component_ordering_key)) logging.info("to new entry '%s'" % get_string_with_no_auth_from_source_entry(entry)) del self.found_components[entry.dist] return foundToDist def updateSourcesList(self): logging.debug("updateSourcesList()") self.sources = SourcesList(matcherPath=".") # backup first! self.sources.backup(self.sources_backup_ext) if not self.rewriteSourcesList(mirror_check=True): logging.error("No valid mirror found") res = self._view.askYesNoQuestion(_("No valid mirror found"), _("While scanning your repository " "information no mirror entry for " "the upgrade was found. " "This can happen if you run an internal " "mirror or if the mirror information is " "out of date.\n\n" "Do you want to rewrite your " "'sources.list' file anyway? If you choose " "'Yes' here it will update all '%s' to '%s' " "entries.\n" "If you select 'No' the upgrade will cancel." ) % (self.fromDist, self.toDist)) if res: # re-init the sources and try again self.sources = SourcesList(matcherPath=".") # its ok if rewriteSourcesList fails here if # we do not use a network, the sources.list may be empty if (not self.rewriteSourcesList(mirror_check=False) and self.useNetwork): #hm, still nothing useful ... prim = _("Generate default sources?") secon = _("After scanning your 'sources.list' no " "valid entry for '%s' was found.\n\n" "Should default entries for '%s' be " "added? If you select 'No', the upgrade " "will cancel.") % (self.fromDist, self.toDist) if not self._view.askYesNoQuestion(prim, secon): self.abort() # add some defaults here # FIXME: find mirror here logging.info("generate new default sources.list") uri = "http://archive.ubuntu.com/ubuntu" comps = ["main","restricted"] self.sources.add("deb", uri, self.toDist, comps) self.sources.add("deb", uri, self.toDist+"-updates", comps) self.sources.add("deb", "http://security.ubuntu.com/ubuntu/", self.toDist+"-security", comps) else: self.abort() # now write self.sources.save() # re-check if the written self.sources are valid, if not revert and # bail out # TODO: check if some main packages are still available or if we # accidentally shot them, if not, maybe offer to write a standard # sources.list? try: sourceslist = apt_pkg.SourceList() sourceslist.read_main_list() except SystemError: logging.error("Repository information invalid after updating (we broke it!)") if os.path.exists("/usr/bin/apport-bug"): self._view.error(_("Repository information invalid"), _("Upgrading the repository information " "resulted in a invalid file so a bug " "reporting process is being started.")) subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"]) else: self._view.error(_("Repository information invalid"), _("Upgrading the repository information " "resulted in a invalid file. To report " "a bug install apport and then execute " "'apport-bug ubuntu-release-upgrader'.")) logging.error("Missing apport-bug, bug report not " "autocreated") return False if self.sources_disabled: self._view.information(_("Third party sources disabled"), _("Some third party entries in your sources.list " "were disabled. You can re-enable them " "after the upgrade with the " "'software-properties' tool or " "your package manager." )) return True def _logChanges(self): # debugging output logging.debug("About to apply the following changes") inst = [] up = [] rm = [] held = [] keep = [] for pkg in self.cache: if pkg.marked_install: inst.append(pkg.name) elif pkg.marked_upgrade: up.append(pkg.name) elif pkg.marked_delete: rm.append(pkg.name) elif (pkg.is_installed and pkg.is_upgradable): held.append(pkg.name) elif pkg.is_installed and pkg.marked_keep: keep.append(pkg.name) logging.debug("Keep at same version: %s" % " ".join(keep)) logging.debug("Upgradable, but held- back: %s" % " ".join(held)) logging.debug("Remove: %s" % " ".join(rm)) logging.debug("Install: %s" % " ".join(inst)) logging.debug("Upgrade: %s" % " ".join(up)) def doPostInitialUpdate(self): # check if we have packages in ReqReinst state that are not # downloadable logging.debug("doPostInitialUpdate") self.quirks.run("PostInitialUpdate") if not self.cache: return False if len(self.cache.req_reinstall_pkgs) > 0: logging.warning("packages in reqReinstall state, trying to fix") self.cache.fix_req_reinst(self._view) self.openCache() if len(self.cache.req_reinstall_pkgs) > 0: reqreinst = self.cache.req_reinstall_pkgs header = ngettext("Package in inconsistent state", "Packages in inconsistent state", len(reqreinst)) summary = ngettext("The package '%s' is in an inconsistent " "state and needs to be reinstalled, but " "no archive can be found for it. " "Please reinstall the package manually " "or remove it from the system.", "The packages '%s' are in an inconsistent " "state and need to be reinstalled, but " "no archive can be found for them. " "Please reinstall the packages manually " "or remove them from the system.", len(reqreinst)) % ", ".join(reqreinst) self._view.error(header, summary) return False # FIXME: check out what packages are downloadable etc to # compare the list after the update again self.obsolete_pkgs = self.cache._getObsoletesPkgs() self.foreign_pkgs = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist) if self.serverMode: self.tasks = self.cache.installedTasks logging.debug("Foreign: %s" % " ".join(self.foreign_pkgs)) logging.debug("Obsolete: %s" % " ".join(self.obsolete_pkgs)) return True def doUpdate(self, showErrors=True, forceRetries=None): logging.debug("running doUpdate() (showErrors=%s)" % showErrors) if not self.useNetwork: logging.debug("doUpdate() will not use the network because self.useNetwork==false") return True self.cache._list.read_main_list() progress = self._view.getAcquireProgress() # FIXME: also remove all files from the lists partial dir! currentRetry = 0 if forceRetries is not None: maxRetries=forceRetries else: maxRetries = self.config.getint("Network","MaxRetries") while currentRetry < maxRetries: try: self.cache.update(progress) except (SystemError, IOError) as e: logging.error("IOError/SystemError in cache.update(): '%s'. Retrying (currentRetry: %s)" % (e,currentRetry)) currentRetry += 1 continue # no exception, so all was fine, we are done return True logging.error("doUpdate() failed completely") if showErrors: self._view.error(_("Error during update"), _("A problem occurred during the update. " "This is usually some sort of network " "problem, please check your network " "connection and retry."), "%s" % e) return False def _checkFreeSpace(self): " this checks if we have enough free space on /var and /usr" err_sum = _("Not enough free disk space") err_long= _("The upgrade has aborted. " "The upgrade needs a total of %s free space on disk '%s'. " "Please free at least an additional %s of disk " "space on '%s'. " "Empty your trash and remove temporary " "packages of former installations using " "'sudo apt-get clean'.") # allow override if self.config.getWithDefault("FreeSpace","SkipCheck",False): logging.warning("free space check skipped via config override") return True # do the check with_snapshots = self._is_apt_btrfs_snapshot_supported() try: self.cache.checkFreeSpace(with_snapshots) except NotEnoughFreeSpaceError as e: # ok, showing multiple error dialog sucks from the UI # perspective, but it means we do not need to break the # string freeze for required in e.free_space_required_list: self._view.error(err_sum, err_long % (required.size_total, required.dir, required.size_needed, required.dir)) return False return True def askDistUpgrade(self): self._view.updateStatus(_("Calculating the changes")) if not self.cache.distUpgrade(self._view, self.serverMode, self._partialUpgrade): return False if self.serverMode: if not self.cache.installTasks(self.tasks): return False # show changes and confirm changes = self.cache.get_changes() self._view.processEvents() # log the changes for debugging self._logChanges() self._view.processEvents() # check if we have enough free space if not self._checkFreeSpace(): return False self._view.processEvents() # get the demotions self.installed_demotions = self.cache.get_installed_demoted_packages() if len(self.installed_demotions) > 0: self.installed_demotions.sort() logging.debug("demoted: '%s'" % " ".join([x.name for x in self.installed_demotions])) logging.debug("found components: %s" % self.found_components) # flush UI self._view.processEvents() # ask the user res = self._view.confirmChanges(_("Do you want to start the upgrade?"), changes, self.installed_demotions, self.cache.required_download) return res def _disableAptCronJob(self): if os.path.exists("/etc/cron.daily/apt"): #self._aptCronJobPerms = os.stat("/etc/cron.daily/apt")[ST_MODE] logging.debug("disabling apt cron job (%s)" % oct(self._aptCronJobPerms)) os.chmod("/etc/cron.daily/apt",0o644) def _enableAptCronJob(self): if os.path.exists("/etc/cron.daily/apt"): logging.debug("enabling apt cron job") os.chmod("/etc/cron.daily/apt", self._aptCronJobPerms) def doDistUpgradeFetching(self): # ensure that no apt cleanup is run during the download/install self._disableAptCronJob() # get the upgrade currentRetry = 0 fprogress = self._view.getAcquireProgress() #iprogress = self._view.getInstallProgress(self.cache) # start slideshow url = self.config.getWithDefault("Distro","SlideshowUrl",None) if url: try: lang = locale.getdefaultlocale()[0].split('_')[0] except: logging.exception("getdefaultlocale") lang = "en" self._view.getHtmlView().open("%s#locale=%s" % (url, lang)) # retry the fetching in case of errors maxRetries = self.config.getint("Network","MaxRetries") # FIXME: we get errors like # "I wasn't able to locate file for the %s package" # here sometimes. its unclear why and not reproducible, the # current theory is that for some reason the file is not # considered trusted at the moment # pkgAcquireArchive::QueueNext() runs debReleaseIndex::IsTrused() # (the later just checks for the existence of the .gpg file) # OR # the fact that we get a pm and fetcher here confuses something # in libapt? # POSSIBLE workaround: keep the list-dir locked so that # no apt-get update can run outside from the release # upgrader user_canceled = False # LP: #1102593 - In Python 3, the targets of except clauses get `del`d # from the current namespace after the exception is handled, so we # must assign it to a different variable in order to use it after # the while loop. exception = None while currentRetry < maxRetries: try: pm = apt_pkg.PackageManager(self.cache._depcache) fetcher = apt_pkg.Acquire(fprogress) self.cache._fetch_archives(fetcher, pm) except apt.cache.FetchCancelledException as e: logging.info("user canceled") user_canceled = True exception = e break except IOError as e: # fetch failed, will be retried logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry)) currentRetry += 1 exception = e continue return True # maximum fetch-retries reached without a successful commit if user_canceled: self._view.information(_("Upgrade canceled"), _("The upgrade will cancel now and the " "original system state will be restored. " "You can resume the upgrade at a later " "time.")) else: logging.error("giving up on fetching after maximum retries") self._view.error(_("Could not download the upgrades"), _("The upgrade has aborted. Please check your " "Internet connection or " "installation media and try again. All files " "downloaded so far have been kept."), "%s" % exception) # abort here because we want our sources.list back self._enableAptCronJob() self.abort() def _is_apt_btrfs_snapshot_supported(self): """ check if apt-btrfs-snapshot is usable """ try: import apt_btrfs_snapshot except ImportError: return try: apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot() res = apt_btrfs.snapshots_supported() except: logging.exception("failed to check btrfs support") return False logging.debug("apt btrfs snapshots supported: %s" % res) return res def _maybe_create_apt_btrfs_snapshot(self): """ create btrfs snapshot (if btrfs layout is there) """ if not self._is_apt_btrfs_snapshot_supported(): return import apt_btrfs_snapshot apt_btrfs = apt_btrfs_snapshot.AptBtrfsSnapshot() prefix = "release-upgrade-%s-" % self.toDist res = apt_btrfs.create_btrfs_root_snapshot(prefix) logging.info("creating snapshot '%s' (success=%s)" % (prefix, res)) def doDistUpgrade(self): # add debug code only here #apt_pkg.config.set("Debug::pkgDpkgPM", "1") #apt_pkg.config.set("Debug::pkgOrderList", "1") #apt_pkg.config.set("Debug::pkgPackageManager", "1") # get the upgrade currentRetry = 0 fprogress = self._view.getAcquireProgress() iprogress = self._view.getInstallProgress(self.cache) # retry the fetching in case of errors maxRetries = self.config.getint("Network","MaxRetries") if not self._partialUpgrade: self.quirks.run("StartUpgrade") # FIXME: take this into account for diskspace calculation self._maybe_create_apt_btrfs_snapshot() res = False while currentRetry < maxRetries: try: res = self.cache.commit(fprogress,iprogress) logging.debug("cache.commit() returned %s" % res) except SystemError as e: logging.error("SystemError from cache.commit(): %s" % e) # if its a ordering bug we can cleanly revert to # the previous release, no packages have been installed # yet (LP: #328655, #356781) if os.path.exists("/var/run/ubuntu-release-upgrader-apt-exception"): e = open("/var/run/ubuntu-release-upgrader-apt-exception").read() logging.error("found exception: '%s'" % e) # if its a ordering bug we can cleanly revert but we need to write # a marker for the parent process to know its this kind of error pre_configure_errors = [ "E:Internal Error, Could not perform immediate configuration", "E:Couldn't configure pre-depend "] for preconf_error in pre_configure_errors: if str(e).startswith(preconf_error): logging.debug("detected preconfigure error, restorting state") self._enableAptCronJob() # FIXME: strings are not good, but we are in string freeze # currently msg = _("Error during commit") msg += "\n'%s'\n" % str(e) msg += _("Restoring original system state") self._view.error(_("Could not install the upgrades"), msg) # abort() exits cleanly self.abort() # invoke the frontend now and show a error message msg = _("The upgrade has aborted. Your system " "could be in an unusable state. A recovery " "will run now (dpkg --configure -a).") if not self._partialUpgrade: if not run_apport(): msg += _("\n\nPlease report this bug in a browser at " "http://bugs.launchpad.net/ubuntu/+source/ubuntu-release-upgrader/+filebug " "and attach the files in /var/log/dist-upgrade/ " "to the bug report.\n" "%s" % e) self._view.error(_("Could not install the upgrades"), msg) # installing the packages failed, can't be retried cmd = ["/usr/bin/dpkg","--configure","-a"] if os.environ.get("DEBIAN_FRONTEND") == "noninteractive": cmd.append("--force-confold") self._view.getTerminal().call(cmd) self._enableAptCronJob() return False except IOError as e: # fetch failed, will be retried logging.error("IOError in cache.commit(): '%s'. Retrying (currentTry: %s)" % (e,currentRetry)) currentRetry += 1 continue except OSError as e: logging.exception("cache.commit()") # deal gracefully with: # OSError: [Errno 12] Cannot allocate memory if e.errno == 12: self._enableAptCronJob() msg = _("Error during commit") msg += "\n'%s'\n" % str(e) msg += _("Restoring original system state") self._view.error(_("Could not install the upgrades"), msg) # abort() exits cleanly self.abort() # no exception, so all was fine, we are done self._enableAptCronJob() return True # maximum fetch-retries reached without a successful commit logging.error("giving up on fetching after maximum retries") self._view.error(_("Could not download the upgrades"), _("The upgrade has aborted. Please check your "\ "Internet connection or "\ "installation media and try again. "), "%s" % e) # abort here because we want our sources.list back self.abort() def doPostUpgrade(self): # reopen cache self.openCache() # run the quirks handler that does does like things adding # missing groups or similar work arounds, only do it on real # upgrades self.quirks.run("PostUpgrade") # check out what packages are cruft now # use self.{foreign,obsolete}_pkgs here and see what changed now_obsolete = self.cache._getObsoletesPkgs() now_foreign = self.cache._getForeignPkgs(self.origin, self.fromDist, self.toDist) logging.debug("Obsolete: %s" % " ".join(now_obsolete)) logging.debug("Foreign: %s" % " ".join(now_foreign)) # now sanity check - if a base meta package is in the obsolete list now, that means # that something went wrong (see #335154) badly with the network. this should never happen, but it did happen # at least once so we add extra paranoia here for pkg in self.config.getlist("Distro","BaseMetaPkgs"): if pkg in now_obsolete: logging.error("the BaseMetaPkg '%s' is in the obsolete list, something is wrong, ignoring the obsoletes" % pkg) now_obsolete = set() break # check if we actually want obsolete removal if not self.config.getWithDefault("Distro","RemoveObsoletes", True): logging.debug("Skipping obsolete Removal") return True # now get the meta-pkg specific obsoletes and purges for pkg in self.config.getlist("Distro","MetaPkgs"): if pkg in self.cache and self.cache[pkg].is_installed: self.forced_obsoletes.extend(self.config.getlist(pkg,"ForcedObsoletes")) # now add the obsolete kernels to the forced obsoletes self.forced_obsoletes.extend(self.cache.identifyObsoleteKernels()) logging.debug("forced_obsoletes: %s", self.forced_obsoletes) # mark packages that are now obsolete (and where not obsolete # before) to be deleted. make sure to not delete any foreign # (that is, not from ubuntu) packages if self.useNetwork: # we can only do the obsoletes calculation here if we use a # network. otherwise after rewriting the sources.list everything # that is not on the CD becomes obsolete (not-downloadable) remove_candidates = now_obsolete - self.obsolete_pkgs else: # initial remove candidates when no network is used should # be the demotions to make sure we don't leave potential # unsupported software remove_candidates = set([p.name for p in self.installed_demotions]) remove_candidates |= set(self.forced_obsoletes) # no go for the unused dependencies unused_dependencies = self.cache._getUnusedDependencies() logging.debug("Unused dependencies: %s" %" ".join(unused_dependencies)) remove_candidates |= set(unused_dependencies) # see if we actually have to do anything here if not self.config.getWithDefault("Distro","RemoveObsoletes", True): logging.debug("Skipping RemoveObsoletes as stated in the config") remove_candidates = set() logging.debug("remove_candidates: '%s'" % remove_candidates) logging.debug("Start checking for obsolete pkgs") progress = self._view.getOpCacheProgress() for (i, pkgname) in enumerate(remove_candidates): progress.update((i/float(len(remove_candidates)))*100.0) if pkgname not in self.foreign_pkgs: self._view.processEvents() if not self.cache.tryMarkObsoleteForRemoval(pkgname, remove_candidates, self.foreign_pkgs): logging.debug("'%s' scheduled for remove but not safe to remove, skipping", pkgname) logging.debug("Finish checking for obsolete pkgs") progress.done() # get changes changes = self.cache.get_changes() logging.debug("The following packages are marked for removal: %s" % " ".join([pkg.name for pkg in changes])) summary = _("Remove obsolete packages?") actions = [_("_Keep"), _("_Remove")] # FIXME Add an explanation about what obsolete packages are #explanation = _("") if (len(changes) > 0 and self._view.confirmChanges(summary, changes, [], 0, actions, False)): fprogress = self._view.getAcquireProgress() iprogress = self._view.getInstallProgress(self.cache) try: self.cache.commit(fprogress,iprogress) except (SystemError, IOError) as e: logging.error("cache.commit() in doPostUpgrade() failed: %s" % e) self._view.error(_("Error during commit"), _("A problem occurred during the clean-up. " "Please see the below message for more " "information. "), "%s" % e) # run stuff after cleanup self.quirks.run("PostCleanup") # run the post upgrade scripts that can do fixup like xorg.conf # fixes etc - only do on real upgrades if not self._partialUpgrade: self.runPostInstallScripts() return True def runPostInstallScripts(self): """ scripts that are run in any case after the distupgrade finished whether or not it was successful """ # now run the post-upgrade fixup scripts (if any) for script in self.config.getlist("Distro","PostInstallScripts"): if not os.path.exists(script): logging.warning("PostInstallScript: '%s' not found" % script) continue logging.debug("Running PostInstallScript: '%s'" % script) try: # work around kde tmpfile problem where it eats permissions check_and_fix_xbit(script) self._view.getTerminal().call([script], hidden=True) except Exception as e: logging.error("got error from PostInstallScript %s (%s)" % (script, e)) def abort(self): """ abort the upgrade, cleanup (as much as possible) """ logging.debug("abort called") if hasattr(self, "sources"): self.sources.restore_backup(self.sources_backup_ext) if hasattr(self, "aptcdrom"): self.aptcdrom.restore_backup(self.sources_backup_ext) # generate a new cache self._view.updateStatus(_("Restoring original system state")) self._view.abort() self.openCache() sys.exit(1) def _checkDep(self, depstr): " check if a given depends can be satisfied " for or_group in apt_pkg.parse_depends(depstr): logging.debug("checking: '%s' " % or_group) for dep in or_group: depname = dep[0] ver = dep[1] oper = dep[2] if depname not in self.cache: logging.error("_checkDep: '%s' not in cache" % depname) return False inst = self.cache[depname] instver = getattr(inst.installed, "version", None) if (instver != None and apt_pkg.check_dep(instver,oper,ver) == True): return True logging.error("depends '%s' is not satisfied" % depstr) return False def checkViewDepends(self): " check if depends are satisfied " logging.debug("checkViewDepends()") res = True # now check if anything from $foo-updates is required depends = self.config.getlist("View","Depends") depends.extend(self.config.getlist(self._view.__class__.__name__, "Depends")) for dep in depends: logging.debug("depends: '%s'", dep) res &= self._checkDep(dep) if not res: # FIXME: instead of error out, fetch and install it # here self._view.error(_("Required depends is not installed"), _("The required dependency '%s' is not " "installed. " % dep)) sys.exit(1) return res def _verifyBackports(self): # run update (but ignore errors in case the countrymirror # substitution goes wrong, real errors will be caught later # when the cache is searched for the backport packages) backportslist = self.config.getlist("PreRequists","Packages") i=0 noCache = apt_pkg.config.find("Acquire::http::No-Cache","false") maxRetries = self.config.getint("Network","MaxRetries") while i < maxRetries: self.doUpdate(showErrors=False) self.openCache() for pkgname in backportslist: if pkgname not in self.cache: logging.error("Can not find backport '%s'" % pkgname) raise NoBackportsFoundException(pkgname) if self._allBackportsAuthenticated(backportslist): break # FIXME: move this to some more generic place logging.debug("setting a cache control header to turn off caching temporarily") apt_pkg.config.set("Acquire::http::No-Cache","true") i += 1 if i == maxRetries: logging.error("pre-requists item is NOT trusted, giving up") return False apt_pkg.config.set("Acquire::http::No-Cache",noCache) return True def _allBackportsAuthenticated(self, backportslist): # check if the user overwrote the check if apt_pkg.config.find_b("APT::Get::AllowUnauthenticated",False) == True: logging.warning("skip authentication check because of APT::Get::AllowUnauthenticated==true") return True try: b = self.config.getboolean("Distro","AllowUnauthenticated") if b: return True except NoOptionError: pass for pkgname in backportslist: pkg = self.cache[pkgname] if not pkg.candidate: return False for cand in pkg.candidate.origins: if cand.trusted: break else: return False return True def isMirror(self, uri): """ check if uri is a known mirror """ # deal with username:password in a netloc raw_uri = uri.rstrip("/") scheme, netloc, path, query, fragment = urlsplit(raw_uri) if "@" in netloc: netloc = netloc.split("@")[1] # construct new mirror url without the username/pw uri = "%s://%s%s" % (scheme, netloc, path) for mirror in self.valid_mirrors: mirror = mirror.rstrip("/") if is_mirror(mirror, uri): return True # deal with mirrors like # deb http://localhost:9977/security.ubuntu.com/ubuntu intrepid-security main restricted # both apt-debtorrent and apt-cacher use this (LP: #365537) mirror_host_part = mirror.split("//")[1] if uri.endswith(mirror_host_part): logging.debug("found apt-cacher/apt-torrent style uri %s" % uri) return True return False def isThirdPartyMirror(self, uri): " check if uri is a whitelisted third-party mirror " uri = uri.rstrip("/") for mirror in self.valid_3p_mirrors: mirror = mirror.rstrip("/") if is_mirror(mirror, uri): return True return False def _getPreReqMirrorLines(self, dumb=False): " get sources.list snippet lines for the current mirror " lines = "" sources = SourcesList(matcherPath=".") for entry in sources.list: if entry.invalid or entry.disabled: continue if (entry.type == "deb" and entry.disabled == False and self.isMirror(entry.uri) and "main" in entry.comps and "%s-updates" % self.fromDist in entry.dist and not entry.uri.startswith("http://security.ubuntu.com") and not entry.uri.startswith("http://archive.ubuntu.com") ): new_line = "deb %s %s-updates main\n" % (entry.uri, self.fromDist) if not new_line in lines: lines += new_line # FIXME: do we really need "dumb" mode? #if (dumb and entry.type == "deb" and # "main" in entry.comps): # lines += "deb %s %s-proposed main\n" % (entry.uri, self.fromDist) return lines def _addPreRequistsSourcesList(self, template, out, dumb=False): " add prerequists based on template into the path outfile " # go over the sources.list and try to find a valid mirror # that we can use to add the backports dir logging.debug("writing prerequists sources.list at: '%s' " % out) outfile = open(out, "w") mirrorlines = self._getPreReqMirrorLines(dumb) for line in open(template): template = Template(line) outline = template.safe_substitute(mirror=mirrorlines) outfile.write(outline) logging.debug("adding '%s' prerequists" % outline) outfile.close() return True def getRequiredBackports(self): " download the backports specified in DistUpgrade.cfg " logging.debug("getRequiredBackports()") res = True backportsdir = os.path.join(os.getcwd(),"backports") if not os.path.exists(backportsdir): os.mkdir(backportsdir) backportslist = self.config.getlist("PreRequists","Packages") # FIXME: this needs to be ported # if we have them on the CD we are fine if self.aptcdrom and not self.useNetwork: logging.debug("Searching for pre-requists on CDROM") p = os.path.join(self.aptcdrom.cdrompath, "dists/stable/main/dist-upgrader/binary-%s/" % apt_pkg.config.find("APT::Architecture")) found_pkgs = set() for deb in glob.glob(p+"*_*.deb"): logging.debug("found pre-req '%s' to '%s'" % (deb, backportsdir)) found_pkgs.add(os.path.basename(deb).split("_")[0]) # now check if we got all backports on the CD if not set(backportslist).issubset(found_pkgs): logging.error("Expected backports: '%s' but got '%s'" % (set(backportslist), found_pkgs)) return False # now install them self.cache.release_lock() p = subprocess.Popen( ["/usr/bin/dpkg", "-i", ] + glob.glob(p+"*_*.deb"), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) res = None while res is None: res = p.poll() self._view.pulseProgress() time.sleep(0.02) self._view.pulseProgress(finished=True) self.cache.get_lock() logging.info("installing backport debs exit code '%s'" % res) logging.debug("dpkg output:\n%s" % p.communicate()[0]) if res != 0: return False # and re-start itself when it done return self.setupRequiredBackports() # we support PreRequists/SourcesList-$arch sections here too # # logic for mirror finding works list this: # - use the mirror template from the config, then: [done] # # - try to find known mirror (isMirror) and prepend it [done] # - archive.ubuntu.com is always a fallback at the end [done] # # see if we find backports with that # - if not, try guessing based on URI, Trust and Dist [done] # in existing sources.list (internal mirror with no # outside connection maybe) # # make sure to remove file on cancel # FIXME: use the DistUpgradeFetcherCore logic # in mirror_from_sources_list() here # (and factor that code out into a helper) conf_option = "SourcesList" if self.config.has_option("PreRequists",conf_option+"-%s" % self.arch): conf_option = conf_option + "-%s" % self.arch prereq_template = self.config.get("PreRequists",conf_option) if not os.path.exists(prereq_template): logging.error("sourceslist not found '%s'" % prereq_template) return False outpath = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template) outfile = os.path.join(apt_pkg.config.find_dir("Dir::Etc::sourceparts"), prereq_template) self._addPreRequistsSourcesList(prereq_template, outfile) try: self._verifyBackports() except NoBackportsFoundException as e: self._addPreRequistsSourcesList(prereq_template, outfile, dumb=True) try: self._verifyBackports() except NoBackportsFoundException as e: logging.warning("no backport for '%s' found" % e) return False # FIXME: sanity check the origin (just for safety) for pkgname in backportslist: pkg = self.cache[pkgname] # look for the right version (backport) ver = self.cache._depcache.get_candidate_ver(pkg._pkg) if not ver: logging.error("No candidate for '%s'" % pkgname) os.unlink(outpath) return False if ver.file_list == None: logging.error("No ver.file_list for '%s'" % pkgname) os.unlink(outpath) return False logging.debug("marking '%s' for install" % pkgname) # mark install pkg.mark_install(auto_inst=False, auto_fix=False) # now get it res = False try: res = self.cache.commit(self._view.getAcquireProgress(), self._view.getInstallProgress(self.cache)) except IOError as e: logging.error("fetch_archives returned '%s'" % e) res = False except SystemError as e: logging.error("install_archives returned '%s'" % e) res = False if res == False: logging.warning("_fetch_archives for backports returned False") # all backports done, remove the pre-requirests.list file again try: os.unlink(outfile) except Exception as e: logging.error("failed to unlink pre-requists file: '%s'" % e) return self.setupRequiredBackports() # used by both cdrom/http fetcher def setupRequiredBackports(self): # ensure that the new release upgrader uses the latest python-apt # from the backport path os.environ["PYTHONPATH"] = "/usr/lib/release-upgrader-python-apt" # copy log so that it gets not overwritten logging.shutdown() shutil.copy("/var/log/dist-upgrade/main.log", "/var/log/dist-upgrade/main_pre_req.log") # now exec self again args = sys.argv + ["--have-prerequists"] if self.useNetwork: args.append("--with-network") else: args.append("--without-network") logging.info("restarting upgrader") #print("restarting upgrader to make use of the backports") # work around kde being clever and removing the x bit check_and_fix_xbit(sys.argv[0]) os.execve(sys.argv[0],args, os.environ) # this is the core def fullUpgrade(self): # sanity check (check for ubuntu-desktop, brokenCache etc) self._view.updateStatus(_("Checking package manager")) self._view.setStep(STEP_PREPARE) if not self.prepare(): logging.error("self.prepared() failed") if os.path.exists("/usr/bin/apport-bug"): self._view.error(_("Preparing the upgrade failed"), _("Preparing the system for the upgrade " "failed so a bug reporting process is " "being started.")) subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"]) else: self._view.error(_("Preparing the upgrade failed"), _("Preparing the system for the upgrade " "failed. To report a bug install apport " "and then execute 'apport-bug " "ubuntu-release-upgrader'.")) logging.error("Missing apport-bug, bug report not " "autocreated") sys.exit(1) # mvo: commented out for now, see #54234, this needs to be # refactored to use a arch=any tarball if (self.config.has_section("PreRequists") and self.options and self.options.havePrerequists == False): logging.debug("need backports") # get backported packages (if needed) if not self.getRequiredBackports(): if os.path.exists("/usr/bin/apport-bug"): self._view.error(_("Getting upgrade prerequisites failed"), _("The system was unable to get the " "prerequisites for the upgrade. " "The upgrade will abort now and restore " "the original system state.\n" "\n" "Additionally, a bug reporting process is " "being started.")) subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"]) else: self._view.error(_("Getting upgrade prerequisites failed"), _("The system was unable to get the " "prerequisites for the upgrade. " "The upgrade will abort now and restore " "the original system state.\n" "\n" "To report a bug install apport and " "then execute 'apport-bug " "ubuntu-release-upgrader'.")) logging.error("Missing apport-bug, bug report not " "autocreated") self.abort() # run a "apt-get update" now, its ok to ignore errors, # because # a) we disable any third party sources later # b) we check if we have valid ubuntu sources later # after we rewrite the sources.list and do a # apt-get update there too # because the (unmodified) sources.list of the user # may contain bad/unreachable entries we run only # with a single retry self.doUpdate(showErrors=False, forceRetries=1) self.openCache() # do pre-upgrade stuff (calc list of obsolete pkgs etc) if not self.doPostInitialUpdate(): self.abort() # update sources.list self._view.setStep(STEP_MODIFY_SOURCES) self._view.updateStatus(_("Updating repository information")) if not self.updateSourcesList(): self.abort() # add cdrom (if we have one) if (self.aptcdrom and not self.aptcdrom.add(self.sources_backup_ext)): self._view.error(_("Failed to add the cdrom"), _("Sorry, adding the cdrom was not successful.")) sys.exit(1) # then update the package index files if not self.doUpdate(): self.abort() # then open the cache (again) self._view.updateStatus(_("Checking package manager")) # if something fails here (e.g. locking the cache) we need to # restore the system state (LP: #1052605) self.openCache(restore_sources_list_on_fail=True) # re-check server mode because we got new packages (it may happen # that the system had no sources.list entries and therefore no # desktop file information) self.serverMode = self.cache.need_server_mode() # do it here as we need to know if we are in server or client mode self.quirks.ensure_recommends_are_installed_on_desktops() # now check if we still have some key packages available/downloadable # after the update - if not something went seriously wrong # (this happend e.g. during the intrepid->jaunty upgrade for some # users when de.archive.ubuntu.com was overloaded) for pkg in self.config.getlist("Distro","BaseMetaPkgs"): if (pkg not in self.cache or not self.cache.anyVersionDownloadable(self.cache[pkg])): # FIXME: we could offer to add default source entries here, # but we need to be careful to not duplicate them # (i.e. the error here could be something else than # missing sources entries but network errors etc) logging.error("No '%s' available/downloadable after sources.list rewrite+update" % pkg) self._view.error(_("Invalid package information"), _("After updating your package " "information, the essential package '%s' " "could not be located. This may be " "because you have no official mirrors " "listed in your software sources, or " "because of excessive load on the mirror " "you are using. See /etc/apt/sources.list " "for the current list of configured " "software sources." "\n" "In the case of an overloaded mirror, you " "may want to try the upgrade again later.") % pkg) if os.path.exists("/usr/bin/apport-bug"): subprocess.Popen(["apport-bug", "ubuntu-release-upgrader-core"]) else: logging.error("Missing apport-bug, bug report not " "autocreated") self.abort() # calc the dist-upgrade and see if the removals are ok/expected # do the dist-upgrade self._view.updateStatus(_("Calculating the changes")) if not self.askDistUpgrade(): self.abort() # fetch the stuff self._view.setStep(STEP_FETCH) self._view.updateStatus(_("Fetching")) if not self.doDistUpgradeFetching(): self.abort() # now do the upgrade self._view.setStep(STEP_INSTALL) self._view.updateStatus(_("Upgrading")) if not self.doDistUpgrade(): # run the post install scripts (for stuff like UUID conversion) self.runPostInstallScripts() # don't abort here, because it would restore the sources.list self._view.information(_("Upgrade complete"), _("The upgrade has completed but there " "were errors during the upgrade " "process.")) sys.exit(1) # do post-upgrade stuff self._view.setStep(STEP_CLEANUP) self._view.updateStatus(_("Searching for obsolete software")) self.doPostUpgrade() # comment out cdrom source if self.aptcdrom: self.aptcdrom.comment_out_cdrom_entry() # remove upgrade-available notice if os.path.exists("/var/lib/ubuntu-release-upgrader/release-upgrade-available"): os.unlink("/var/lib/ubuntu-release-upgrader/release-upgrade-available") # done, ask for reboot self._view.setStep(STEP_REBOOT) self._view.updateStatus(_("System upgrade is complete.")) # FIXME should we look into /var/run/reboot-required here? if (not inside_chroot() and self._view.confirmRestart()): subprocess.Popen("/sbin/reboot") sys.exit(0) return True def run(self): self._view.processEvents() return self.fullUpgrade() def doPartialUpgrade(self): " partial upgrade mode, useful for repairing " self._view.setStep(STEP_PREPARE) self._view.hideStep(STEP_MODIFY_SOURCES) self._view.hideStep(STEP_REBOOT) self._partialUpgrade = True self.prepare() if not self.doPostInitialUpdate(): return False if not self.askDistUpgrade(): return False self._view.setStep(STEP_FETCH) self._view.updateStatus(_("Fetching")) if not self.doDistUpgradeFetching(): return False self._view.setStep(STEP_INSTALL) self._view.updateStatus(_("Upgrading")) if not self.doDistUpgrade(): self._view.information(_("Upgrade complete"), _("The upgrade has completed but there " "were errors during the upgrade " "process.")) return False self._view.setStep(STEP_CLEANUP) if not self.doPostUpgrade(): self._view.information(_("Upgrade complete"), _("The upgrade has completed but there " "were errors during the upgrade " "process.")) return False if os.path.exists(REBOOT_REQUIRED_FILE): # we can not talk to session management here, we run as root if self._view.confirmRestart(): subprocess.Popen("/sbin/reboot") else: self._view.information(_("Upgrade complete"), _("The partial upgrade was completed.")) return True if __name__ == "__main__": from .DistUpgradeViewText import DistUpgradeViewText logging.basicConfig(level=logging.DEBUG) v = DistUpgradeViewText() dc = DistUpgradeController(v) #dc.openCache() dc._disableAptCronJob() dc._enableAptCronJob() #dc._addRelatimeToFstab() #dc.prepare() #dc.askDistUpgrade() #dc._checkFreeSpace() #dc._rewriteFstab() #dc._checkAdminGroup() #dc._rewriteAptPeriodic(2)