From 5f21f43af753b3a0aebc8c0e6e82e2ea6321bc74 Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 10 Jul 2016 13:32:43 -0400 Subject: [PATCH 1/3] #22758: fix regression in handling of secure cookies. This backports the fix from #16611, per discussion with the release manager. --- Lib/http/cookies.py | 29 ++++++++++++++++--------- Lib/test/test_http_cookies.py | 40 ++++++++++++++++++++++++++++++++++- Misc/NEWS | 3 +++ 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 50aabd60d2a..c2e311dbc18 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -338,6 +338,8 @@ class Morsel(dict): "version" : "Version", } + _flags = {'secure', 'httponly'} + def __init__(self): # Set defaults self.key = self.value = self.coded_value = None @@ -437,15 +439,18 @@ _CookiePattern = re.compile(r""" (?P # Start of group 'key' [""" + _LegalKeyChars + r"""]+? # Any word of at least one letter ) # End of group 'key' - \s*=\s* # Equal Sign - (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string - | # or + ( # Optional group: there may not be a value. + \s*=\s* # Equal Sign + (?P # Start of group 'val' + "(?:[^\\"]|\\.)*" # Any doublequoted string + | # or \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr - | # or - [""" + _LegalValueChars + r"""]* # Any word or empty string - ) # End of group 'val' - \s*;? # Probably ending in a semi-colon + | # or + [""" + _LegalValueChars + r"""]* # Any word or empty string + ) # End of group 'val' + )? # End of optional value group + \s* # Any number of spaces. + (\s+|;|$) # Ending either at space, semicolon, or EOS. """, re.ASCII) # May be removed if safe. @@ -551,8 +556,12 @@ class BaseCookie(dict): M[key[1:]] = value elif key.lower() in Morsel._reserved: if M: - M[key] = _unquote(value) - else: + if value is None: + if key.lower() in Morsel._flags: + M[key] = True + else: + M[key] = _unquote(value) + elif value is not None: rval, cval = self.value_decode(value) self.__set(key, rval, cval) M = self[key] diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index a0edcbfc901..9340c0e3cb0 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -114,13 +114,51 @@ class CookieTests(unittest.TestCase): self.assertEqual(C.output(), 'Set-Cookie: Customer="WILE_E_COYOTE"; Max-Age=10') - # others + def test_set_secure_httponly_attrs(self): C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"') C['Customer']['secure'] = True C['Customer']['httponly'] = True self.assertEqual(C.output(), 'Set-Cookie: Customer="WILE_E_COYOTE"; httponly; secure') + def test_secure_httponly_false_if_not_present(self): + C = cookies.SimpleCookie() + C.load('eggs=scrambled; Path=/bacon') + self.assertFalse(C['eggs']['httponly']) + self.assertFalse(C['eggs']['secure']) + + def test_secure_httponly_true_if_present(self): + # Issue 16611 + C = cookies.SimpleCookie() + C.load('eggs=scrambled; httponly; secure; Path=/bacon') + self.assertTrue(C['eggs']['httponly']) + self.assertTrue(C['eggs']['secure']) + + def test_secure_httponly_true_if_have_value(self): + # This isn't really valid, but demonstrates what the current code + # is expected to do in this case. + C = cookies.SimpleCookie() + C.load('eggs=scrambled; httponly=foo; secure=bar; Path=/bacon') + self.assertTrue(C['eggs']['httponly']) + self.assertTrue(C['eggs']['secure']) + # Here is what it actually does; don't depend on this behavior. These + # checks are testing backward compatibility for issue 16611. + self.assertEqual(C['eggs']['httponly'], 'foo') + self.assertEqual(C['eggs']['secure'], 'bar') + + def test_bad_attrs(self): + # issue 16611: make sure we don't break backward compatibility. + C = cookies.SimpleCookie() + C.load('cookie=with; invalid; version; second=cookie;') + self.assertEqual(C.output(), + 'Set-Cookie: cookie=with\r\nSet-Cookie: second=cookie') + + def test_extra_spaces(self): + C = cookies.SimpleCookie() + C.load('eggs = scrambled ; secure ; path = bar ; foo=foo ') + self.assertEqual(C.output(), + 'Set-Cookie: eggs=scrambled; Path=bar; secure\r\nSet-Cookie: foo=foo') + def test_quoted_meta(self): # Try cookie with quoted meta-data C = cookies.SimpleCookie() diff --git a/Misc/NEWS b/Misc/NEWS index df54574058b..b595a034f82 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -28,6 +28,9 @@ Tests self-signed.pythontest.net. This avoids relying on svn.python.org, which recently changed root certificate. +- Issue #22758: Fix a regression that no longer allowed secure and httponly + cookies to be parsed. + What's New in Python 3.2.6? =========================== From 9bdb1edf35a5ee96f89973f55c52bb7da86dbd9e Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Sun, 10 Jul 2016 13:46:34 -0400 Subject: [PATCH 2/3] Issue #27173: Add 'IDLE Modern Unix' to the built-in key sets. Make the default key set depend on the platform. Add tests for changes to the config module. --- Lib/idlelib/config-keys.def | 51 +++++++++++ Lib/idlelib/config-main.def | 4 +- Lib/idlelib/config.py | 130 +++++++++++++++++---------- Lib/idlelib/configdialog.py | 44 +++++++-- Lib/idlelib/idle_test/test_config.py | 98 ++++++++++++++++++++ 5 files changed, 270 insertions(+), 57 deletions(-) create mode 100644 Lib/idlelib/idle_test/test_config.py diff --git a/Lib/idlelib/config-keys.def b/Lib/idlelib/config-keys.def index 3bfcb690153..64788f9adf0 100644 --- a/Lib/idlelib/config-keys.def +++ b/Lib/idlelib/config-keys.def @@ -109,6 +109,57 @@ change-indentwidth= del-word-left= del-word-right= +[IDLE Modern Unix] +copy = +cut = +paste = +beginning-of-line = +center-insert = +close-all-windows = +close-window = +do-nothing = +end-of-file = +history-next = +history-previous = +interrupt-execution = +view-restart = +restart-shell = +open-class-browser = +open-module = +open-new-window = +open-window-from-file = +plain-newline-and-indent = +print-window = +python-context-help = +python-docs = +redo = +remove-selection = +save-copy-of-window-as-file = +save-window-as-file = +save-window = +select-all = +toggle-auto-coloring = +undo = +find = +find-again = +find-in-files = +find-selection = +replace = +goto-line = +smart-backspace = +newline-and-indent = +smart-indent = +indent-region = +dedent-region = +comment-region = +uncomment-region = +tabify-region = +untabify-region = +toggle-tabs = +change-indentwidth = +del-word-left = +del-word-right = + [IDLE Classic Mac] copy= cut= diff --git a/Lib/idlelib/config-main.def b/Lib/idlelib/config-main.def index 8ebbc1b4c22..a61bba7ef35 100644 --- a/Lib/idlelib/config-main.def +++ b/Lib/idlelib/config-main.def @@ -70,7 +70,9 @@ name2= [Keys] default= 1 -name= IDLE Classic Windows +name= +name2= +# name2 set in user config-main.cfg for keys added after 2016 July 1 [History] cyclic=1 diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index 51ef21b1070..f2437a86317 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -234,10 +234,7 @@ class IdleConf: ' from section %r: %r' % (type, option, section, self.userCfg[configType].Get(section, option, raw=raw))) - try: - print(warning, file=sys.stderr) - except OSError: - pass + _warn(warning, configType, section, option) try: if self.defaultCfg[configType].has_option(section,option): return self.defaultCfg[configType].Get( @@ -251,10 +248,7 @@ class IdleConf: ' from section %r.\n' ' returning default value: %r' % (option, section, default)) - try: - print(warning, file=sys.stderr) - except OSError: - pass + _warn(warning, configType, section, option) return default def SetOption(self, configType, section, option, value): @@ -362,47 +356,68 @@ class IdleConf: '\n from theme %r.\n' ' returning default color: %r' % (element, themeName, theme[element])) - try: - print(warning, file=sys.stderr) - except OSError: - pass + _warn(warning, 'highlight', themeName, element) theme[element] = cfgParser.Get( themeName, element, default=theme[element]) return theme def CurrentTheme(self): - """Return the name of the currently active text color theme. + "Return the name of the currently active text color theme." + return self.current_colors_and_keys('Theme') + + def CurrentKeys(self): + """Return the name of the currently active key set.""" + return self.current_colors_and_keys('Keys') + + def current_colors_and_keys(self, section): + """Return the currently active name for Theme or Keys section. + + idlelib.config-main.def ('default') includes these sections - idlelib.config-main.def includes this section [Theme] default= 1 name= IDLE Classic name2= - # name2 set in user config-main.cfg for themes added after 2015 Oct 1 - Item name2 is needed because setting name to a new builtin - causes older IDLEs to display multiple error messages or quit. + [Keys] + default= 1 + name= + name2= + + Item 'name2', is used for built-in ('default') themes and keys + added after 2015 Oct 1 and 2016 July 1. This kludge is needed + because setting 'name' to a builtin not defined in older IDLEs + to display multiple error messages or quit. See https://bugs.python.org/issue25313. - When default = True, name2 takes precedence over name, - while older IDLEs will just use name. + When default = True, 'name2' takes precedence over 'name', + while older IDLEs will just use name. When default = False, + 'name2' may still be set, but it is ignored. """ + cfgname = 'highlight' if section == 'Theme' else 'keys' default = self.GetOption('main', 'Theme', 'default', type='bool', default=True) + name = '' if default: - theme = self.GetOption('main', 'Theme', 'name2', default='') - if default and not theme or not default: - theme = self.GetOption('main', 'Theme', 'name', default='') - source = self.defaultCfg if default else self.userCfg - if source['highlight'].has_section(theme): - return theme + name = self.GetOption('main', section, 'name2', default='') + if not name: + name = self.GetOption('main', section, 'name', default='') + if name: + source = self.defaultCfg if default else self.userCfg + if source[cfgname].has_section(name): + return name + return "IDLE Classic" if section == 'Theme' else self.default_keys() + + @staticmethod + def default_keys(): + if sys.platform[:3] == 'win': + return 'IDLE Classic Windows' + elif sys.platform == 'darwin': + return 'IDLE Classic OSX' else: - return "IDLE Classic" + return 'IDLE Modern Unix' - def CurrentKeys(self): - "Return the name of the currently active key set." - return self.GetOption('main', 'Keys', 'name', default='') - - def GetExtensions(self, active_only=True, editor_only=False, shell_only=False): + def GetExtensions(self, active_only=True, + editor_only=False, shell_only=False): """Return extensions in default and user config-extensions files. If active_only True, only return active (enabled) extensions @@ -422,7 +437,7 @@ class IdleConf: if self.GetOption('extensions', extn, 'enable', default=True, type='bool'): #the extension is enabled - if editor_only or shell_only: # TODO if both, contradictory + if editor_only or shell_only: # TODO both True contradict if editor_only: option = "enable_editor" else: @@ -527,7 +542,8 @@ class IdleConf: eventStr - virtual event, including brackets, as in '<>'. """ eventName = eventStr[2:-2] #trim off the angle brackets - binding = self.GetOption('keys', keySetName, eventName, default='').split() + binding = self.GetOption('keys', keySetName, eventName, default='', + warn_on_default=False).split() return binding def GetCurrentKeySet(self): @@ -638,20 +654,28 @@ class IdleConf: '<>': [''] } if keySetName: - for event in keyBindings: - binding = self.GetKeyBinding(keySetName, event) - if binding: - keyBindings[event] = binding - else: #we are going to return a default, print warning - warning=('\n Warning: config.py - IdleConf.GetCoreKeys' - ' -\n problem retrieving key binding for event %r' - '\n from key set %r.\n' - ' returning default value: %r' % - (event, keySetName, keyBindings[event])) - try: - print(warning, file=sys.stderr) - except OSError: - pass + if not (self.userCfg['keys'].has_section(keySetName) or + self.defaultCfg['keys'].has_section(keySetName)): + warning = ( + '\n Warning: config.py - IdleConf.GetCoreKeys -\n' + ' key set %r is not defined, using default bindings.' % + (keySetName,) + ) + _warn(warning, 'keys', keySetName) + else: + for event in keyBindings: + binding = self.GetKeyBinding(keySetName, event) + if binding: + keyBindings[event] = binding + else: #we are going to return a default, print warning + warning = ( + '\n Warning: config.py - IdleConf.GetCoreKeys -\n' + ' problem retrieving key binding for event %r\n' + ' from key set %r.\n' + ' returning default value: %r' % + (event, keySetName, keyBindings[event]) + ) + _warn(warning, 'keys', keySetName, event) return keyBindings def GetExtraHelpSourceList(self, configSet): @@ -735,6 +759,18 @@ class IdleConf: idleConf = IdleConf() + +_warned = set() +def _warn(msg, *key): + key = (msg,) + key + if key not in _warned: + try: + print(msg, file=sys.stderr) + except OSError: + pass + _warned.add(key) + + # TODO Revise test output, write expanded unittest # if __name__ == '__main__': diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 388b48f088e..fda655f5d72 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -341,6 +341,7 @@ class ConfigDialog(Toplevel): buttonSaveCustomKeys = Button( frames[1], text='Save as New Custom Key Set', command=self.SaveAsNewKeySet) + self.new_custom_keys = Label(frames[0], bd=2) ##widget packing #body @@ -361,6 +362,7 @@ class ConfigDialog(Toplevel): self.radioKeysCustom.grid(row=1, column=0, sticky=W+NS) self.optMenuKeysBuiltin.grid(row=0, column=1, sticky=NSEW) self.optMenuKeysCustom.grid(row=1, column=1, sticky=NSEW) + self.new_custom_keys.grid(row=0, column=2, sticky=NSEW, padx=5, pady=5) self.buttonDeleteCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) buttonSaveCustomKeys.pack(side=LEFT, fill=X, expand=True, padx=2) frames[0].pack(side=TOP, fill=BOTH, expand=True) @@ -514,10 +516,11 @@ class ConfigDialog(Toplevel): self.OnNewColourSet() def VarChanged_builtinTheme(self, *params): + oldthemes = ('IDLE Classic', 'IDLE New') value = self.builtinTheme.get() - if value == 'IDLE Dark': - if idleConf.GetOption('main', 'Theme', 'name') != 'IDLE New': - self.AddChangedItem('main', 'Theme', 'name', 'IDLE Classic') + if value not in oldthemes: + if idleConf.GetOption('main', 'Theme', 'name') not in oldthemes: + self.AddChangedItem('main', 'Theme', 'name', oldthemes[0]) self.AddChangedItem('main', 'Theme', 'name2', value) self.new_custom_theme.config(text='New theme, see Help', fg='#500000') @@ -557,8 +560,23 @@ class ConfigDialog(Toplevel): self.AddChangedItem('extensions', extKeybindSection, event, value) def VarChanged_builtinKeys(self, *params): + oldkeys = ( + 'IDLE Classic Windows', + 'IDLE Classic Unix', + 'IDLE Classic Mac', + 'IDLE Classic OSX', + ) value = self.builtinKeys.get() - self.AddChangedItem('main', 'Keys', 'name', value) + if value not in oldkeys: + if idleConf.GetOption('main', 'Keys', 'name') not in oldkeys: + self.AddChangedItem('main', 'Keys', 'name', oldkeys[0]) + self.AddChangedItem('main', 'Keys', 'name2', value) + self.new_custom_keys.config(text='New key set, see Help', + fg='#500000') + else: + self.AddChangedItem('main', 'Keys', 'name', value) + self.AddChangedItem('main', 'Keys', 'name2', '') + self.new_custom_keys.config(text='', fg='black') self.LoadKeysList(value) def VarChanged_customKeys(self, *params): @@ -767,8 +785,10 @@ class ConfigDialog(Toplevel): else: self.optMenuKeysCustom.SetMenu(itemList, itemList[0]) #revert to default key set - self.keysAreBuiltin.set(idleConf.defaultCfg['main'].Get('Keys', 'default')) - self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name')) + self.keysAreBuiltin.set(idleConf.defaultCfg['main'] + .Get('Keys', 'default')) + self.builtinKeys.set(idleConf.defaultCfg['main'].Get('Keys', 'name') + or idleConf.default_keys()) #user can't back out of these changes, they must be applied now self.SaveAllChangedConfigs() self.ActivateConfigChanges() @@ -1067,7 +1087,7 @@ class ConfigDialog(Toplevel): self.optMenuKeysCustom.SetMenu(itemList, currentOption) itemList = idleConf.GetSectionList('default', 'keys') itemList.sort() - self.optMenuKeysBuiltin.SetMenu(itemList, itemList[0]) + self.optMenuKeysBuiltin.SetMenu(itemList, idleConf.default_keys()) self.SetKeysType() ##load keyset element list keySetName = idleConf.CurrentKeys() @@ -1369,12 +1389,18 @@ machine. Some do not take affect until IDLE is restarted. [Cancel] only cancels changes made since the last save. ''' help_pages = { - 'Highlighting':''' + 'Highlighting': ''' Highlighting: The IDLE Dark color theme is new in October 2015. It can only be used with older IDLE releases if it is saved as a custom theme, with a different name. -''' +''', + 'Keys': ''' +Keys: +The IDLE Modern Unix key set is new in June 2016. It can only +be used with older IDLE releases if it is saved as a custom +key set, with a different name. +''', } diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py new file mode 100644 index 00000000000..bb7732cf7ce --- /dev/null +++ b/Lib/idlelib/idle_test/test_config.py @@ -0,0 +1,98 @@ +'''Test idlelib.config. + +Much is tested by opening config dialog live or in test_configdialog. +Coverage: 27% +''' +from sys import modules +from test.support import captured_stderr +from tkinter import Tk +import unittest +from idlelib import config + +# Tests should not depend on fortuitous user configurations. +# They must not affect actual user .cfg files. +# Replace user parsers with empty parsers that cannot be saved. + +idleConf = config.idleConf +usercfg = idleConf.userCfg +testcfg = {} +usermain = testcfg['main'] = config.IdleUserConfParser('') # filename +userhigh = testcfg['highlight'] = config.IdleUserConfParser('') +userkeys = testcfg['keys'] = config.IdleUserConfParser('') + +def setUpModule(): + idleConf.userCfg = testcfg + +def tearDownModule(): + idleConf.userCfg = testcfg + + +class CurrentColorKeysTest(unittest.TestCase): + """Test correct scenarios for colorkeys and wrap functions. + + The 5 correct patterns are possible results of config dialog. + """ + colorkeys = idleConf.current_colors_and_keys + + def test_old_default(self): + # name2 must be blank + usermain.read_string(''' + [Theme] + default= 1 + ''') + self.assertEqual(self.colorkeys('Theme'), 'IDLE Classic') + usermain['Theme']['name'] = 'IDLE New' + self.assertEqual(self.colorkeys('Theme'), 'IDLE New') + usermain['Theme']['name'] = 'non-default' # error + self.assertEqual(self.colorkeys('Theme'), 'IDLE Classic') + usermain.remove_section('Theme') + + def test_new_default(self): + # name2 overrides name + usermain.read_string(''' + [Theme] + default= 1 + name= IDLE New + name2= IDLE Dark + ''') + self.assertEqual(self.colorkeys('Theme'), 'IDLE Dark') + usermain['Theme']['name2'] = 'non-default' # error + self.assertEqual(self.colorkeys('Theme'), 'IDLE Classic') + usermain.remove_section('Theme') + + def test_user_override(self): + # name2 does not matter + usermain.read_string(''' + [Theme] + default= 0 + name= Custom Dark + ''') # error until set userhigh + self.assertEqual(self.colorkeys('Theme'), 'IDLE Classic') + userhigh.read_string('[Custom Dark]\na=b') + self.assertEqual(self.colorkeys('Theme'), 'Custom Dark') + usermain['Theme']['name2'] = 'IDLE Dark' + self.assertEqual(self.colorkeys('Theme'), 'Custom Dark') + usermain.remove_section('Theme') + userhigh.remove_section('Custom Dark') + + +class WarningTest(unittest.TestCase): + + def test_warn(self): + Equal = self.assertEqual + config._warned = set() + with captured_stderr() as stderr: + config._warn('warning', 'key') + Equal(config._warned, {('warning','key')}) + Equal(stderr.getvalue(), 'warning'+'\n') + with captured_stderr() as stderr: + config._warn('warning', 'key') + Equal(stderr.getvalue(), '') + with captured_stderr() as stderr: + config._warn('warn2', 'yek') + Equal(config._warned, {('warning','key'), ('warn2','yek')}) + Equal(stderr.getvalue(), 'warn2'+'\n') + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 7a139c50dbf98f3f064d0028aeca76952d8785db Mon Sep 17 00:00:00 2001 From: R David Murray Date: Sun, 10 Jul 2016 13:59:01 -0400 Subject: [PATCH 3/3] #26176: fix usage of Address constructor in email examples. Patch by Nathan Harold. --- Doc/includes/email-alternative-new-api.py | 6 +++--- Doc/library/email-examples.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/includes/email-alternative-new-api.py b/Doc/includes/email-alternative-new-api.py index c1255a68dfb..321f727c313 100644 --- a/Doc/includes/email-alternative-new-api.py +++ b/Doc/includes/email-alternative-new-api.py @@ -9,9 +9,9 @@ from email.utils import make_msgid # Create the base text message. msg = EmailMessage() msg['Subject'] = "Ayons asperges pour le déjeuner" -msg['From'] = Address("Pepé Le Pew", "pepe@example.com") -msg['To'] = (Address("Penelope Pussycat", "penelope@example.com"), - Address("Fabrette Pussycat", "fabrette@example.com")) +msg['From'] = Address("Pepé Le Pew", "pepe", "example.com") +msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"), + Address("Fabrette Pussycat", "fabrette", "example.com")) msg.set_content("""\ Salut! diff --git a/Doc/library/email-examples.rst b/Doc/library/email-examples.rst index cbbcb78e245..ca085868b1d 100644 --- a/Doc/library/email-examples.rst +++ b/Doc/library/email-examples.rst @@ -61,7 +61,7 @@ way we could process it: Up to the prompt, the output from the above is:: - To: Penelope Pussycat <"penelope@example.com">, Fabrette Pussycat <"fabrette@example.com"> + To: Penelope Pussycat , Fabrette Pussycat From: Pepé Le Pew Subject: Ayons asperges pour le déjeuner